def plot_features(self, subset, states=None, idx=0):
# plot all states by default
if states is None:
states = [
"rpm",
"rpm_delta",
"cmd",
"cmd_delta",
"height",
"vel",
"acc",
"angles",
"rates",
]
# set up figure
n_rows = len(states) + 1
fig = plt.figure(figsize=(8, 3 * n_rows), constrained_layout=True)
gs = fig.add_gridspec(n_rows, 1)
# load spectrum
dir_spectrum = os.path.join(self._dir_root_set, "Dataset", subset,
"Spectra")
files_spectrum = retrieve_files(dir_spectrum)
Z = pd.read_csv(files_spectrum[idx], header=None).to_numpy()
# set plot title
if self._feature["feature"] == "Mfcc":
title = "MFCC (%d bins)" % self._feature["mfc_coefficients"]
elif self._feature["feature"] == "Stft":
title = "Spectrogram (%d frequency bins)" % self._feature[
"frequency_bins"]
else:
title = "%s-spectrogram (%d frequency bins)" % (
self._feature["feature"],
self._feature["frequency_bins"],
)
# plot spectrum
ax = fig.add_subplot(gs[0])
ax.set_title(title)
ax = ph.plot_spectrum(Z, self._feature)
# load states
dir_states = os.path.join(self._dir_root_set, "Dataset", subset,
"States")
files_states = retrieve_files(dir_states)
S = pd.read_csv(files_states[idx], header=None).to_numpy()
# plot relevant states
colors = ["orangered", "darkolivegreen", "steelblue", "goldenrod"]
for i, state_name in enumerate(states):
ax = fig.add_subplot(gs[1 + i])
ax = ph.plot_states_synchronized(S, state_name, self._feature,
colors)
plt.show()
return fig
def _plot_model_output(self, original, predicted, residual):
fig = plt.figure(figsize=(6, 6))
# plot original
plt.subplot(3, 1, 1)
plt.title("Original spectrum")
ph.plot_spectrum(original, self._spectrum, colorbar=False)
# plot predicted
plt.subplot(3, 1, 2)
plt.title("Predicted spectrum")
ph.plot_spectrum(predicted, self._spectrum, colorbar=False)
# plot residual
plt.subplot(3, 1, 3)
plt.title("Residual spectrum")
ph.plot_spectrum(
residual,
self._spectrum,
colormap="coolwarm",
colorbar=False,
)
fig.tight_layout()
plt.show()
fig.savefig("plottyplot.eps", format="eps")
def plot_spectra_denoised(
self,
noise_ratio,
categories=None,
idx=0,
plot_clean=True,
plot_noise=True,
plot_mixed=True,
plot_predicted=True,
enp_model_index=0,
):
"""Plot a selection of spectra of a denoised test set.
Keyword arguments:
set_name -- set to be plotted (e.g. 'Test'),
noise_ratio -- the ratio of the noise compared to the signal,
categories -- iterable containing the original categories
(airplane, engine, etc.) to be plotted (default: all),
features -- iterable containing the features to be plotted
(default: all),
idx -- index of the feature selected for plotting (default: 0).
plot_clean -- whether to plot the clean spectra (default: True),
plot_noise -- whether to plot the noise-only spectra
(default: True)
plot_mixed -- whether to plot the mixed-only spectra
(default: True)
plot_predicted: whether to plot the predicted ego-noise spectra
(default: True)
enp_model_index -- selects which ego-noise predictor (enp) to use
for the prediction, if multiple are available (default: 0),
colorbar -- whether to plot a colorbar next to each plotted
spectrum (default: False)
Example usage:
plot_spectra_denoised(1.0, ['airplane', 'helicopter'], ['Stft'],
plot_noise=False, plot_predicted=False)
This plots the clean, mixed and denoised spectra of the first
spectrogram belonging to the airplane and helicopter categories
within the test set with a noise ratio of 1.00.
"""
# define directories
dir_denoised = os.path.join(
self._dir_root_ac,
"Features",
"Denoised",
"Spectra",
"Ratio_%.2f" % noise_ratio,
"Test",
)
if plot_clean:
dir_clean = os.path.join(
self._dir_root_ac,
"Features",
"Clean",
"Spectra",
"Test",
)
if plot_noise:
# load noise files
dir_noise = os.path.join(
self._dir_root_enp,
"Dataset",
"Test",
"Spectra",
)
files_noise = retrieve_files(dir_noise)
if plot_mixed:
dir_mixed = os.path.join(
self._dir_root_ac,
"Features",
"Mixed",
"Spectra",
"Ratio_%.2f" % noise_ratio,
"Test",
)
if plot_predicted:
# select appropriate model
dir_model_enp = sorted(
os.listdir(os.path.join(self._dir_root_enp,
"Models")))[enp_model_index]
dir_predicted = os.path.join(
self._dir_root_enp,
"Models",
dir_model_enp,
"Output",
"Test",
"Predicted",
)
files_predicted = retrieve_files(dir_predicted)
# get model context for offset
context = int(dir_model_enp.split("_")[-1][-1])
if plot_noise or plot_predicted:
# also load states files to recover noise indices
dir_states = os.path.join(
self._dir_root_ac,
"Features",
"Mixed",
"States",
"Test",
)
files_states = retrieve_files(dir_states)
# plot all categories if not given
filenames = [f for f in sorted(os.listdir(dir_mixed))]
if categories is None:
file_categories = [f.split("_")[0] for f in filenames]
categories = sorted(list(set(file_categories)))
# setup figure, subfigures
n_categories = len(categories)
n_variants = (1 + int(plot_clean) + int(plot_noise) + int(plot_mixed) +
int(plot_predicted))
fig = plt.figure(
figsize=(6 * n_categories, 4 * n_variants),
constrained_layout=True,
)
gs = fig.add_gridspec(n_variants * 2, n_categories)
# loop through categories
for i, cat in enumerate(categories):
# get filename
fn = [f for f in filenames if f.split("_")[0] == cat][idx]
# load denoised file
file_denoised = os.path.join(dir_denoised, fn)
D = pd.read_csv(file_denoised, header=None).to_numpy()
# store spectra and titles
spectra = []
titles = []
if plot_clean:
# load clean file
file_clean = os.path.join(dir_clean, fn)
C = pd.read_csv(file_clean, header=None).to_numpy()
# add to lists
spectra.append(C)
titles.append("'%s': Clean Sound (%s)" %
(fn, self._feature["feature"]))
if plot_noise or plot_predicted:
# get filename of states file belonging to noise file
fn_states = os.path.split([
f for f in files_states if fn.replace(".csv", "") in f
][0])[-1]
# get file idx, frame idx from states filename for noise file
idx_file = int(fn_states.split("_")[-2])
idx_frame = int(fn_states.split("_")[-1].split(".")[0])
if plot_noise:
# load noise file from idx_file
file_noise = files_noise[idx_file]
# get noise fragment from idx_frame
N = pd.read_csv(file_noise,
header=None).to_numpy()[:,
idx_frame:idx_frame +
D.shape[1]]
# add deltas to noise fragment
N = np.concatenate(
(N, librosa.feature.delta(N, mode="mirror")), axis=0)
# add to lists
spectra.append(N)
titles.append("MAV Noise (%s)" % self._feature["feature"])
if plot_mixed:
# load mixed file
file_mixed = os.path.join(dir_mixed, fn)
M = pd.read_csv(file_mixed, header=None).to_numpy()
# add to lists
spectra.append(M)
titles.append("'%s': Noisy Mix (%s)" %
(fn, self._feature["feature"]))
if plot_predicted:
# load predicted file
file_pred = files_predicted[idx_file]
P = pd.read_csv(file_pred,
header=None).to_numpy()[:, idx_frame -
context:idx_frame -
context + D.shape[1], ]
# add deltas to predicted fragment
P = np.concatenate(
(P, librosa.feature.delta(P, mode="mirror")), axis=0)
spectra.append(P)
titles.append("Predicted MAV Noise (%s)" %
self._feature["feature"])
# add denoised file to end of lists
spectra.append(D)
titles.append("'%s': Denoised Sound (%s)" %
(fn, self._feature["feature"]))
for j, Z in enumerate(spectra):
# plot spectrum
ax = fig.add_subplot(gs[2 * j, i])
ph.plot_spectrum(Z[:Z.shape[0] // 2], self._feature)
ax.set_title(titles[j])
# plot delta-spectrum
ax = fig.add_subplot(gs[2 * j + 1, i])
ph.plot_spectrum(Z[Z.shape[0] // 2:],
self._feature,
colormap="coolwarm")
def plot_spectra_mixed(
self,
set_name,
noise_ratio,
categories=None,
idx=0,
plot_clean=True,
plot_noise=True,
):
"""Plot a selection of spectra of a mixed dataset.
Keyword arguments:
set_name -- set to be plotted (e.g. 'Test'),
noise_ratio -- the ratio of the noise compared to the signal,
categories -- iterable containing the original categories
(airplane, engine, etc.) to be plotted (default: all),
features -- iterable containing the features to be plotted
(default: all),
idx -- index of the feature selected for plotting (default: 0).
plot_clean -- whether to plot the clean spectra (default: True),
plot_noise -- whether to plot the noise-only spectra
(default: True)
colorbar -- whether to plot a colorbar next to each plotted
spectrum (default: False)
Example usage:
plot_spectra_mixed('Train', )...
"""
# define directories
dir_mixed = os.path.join(
self._dir_root_ac,
"Features",
"Mixed",
"Spectra",
"Ratio_%.2f" % noise_ratio,
set_name,
)
if plot_clean:
dir_clean = os.path.join(self._dir_root_ac, "Features", "Clean",
"Spectra", set_name)
if plot_noise:
# load noise files
dir_noise = os.path.join(self._dir_root_enp, "Dataset",
set_name.split(" ")[0], "Spectra")
files_noise = retrieve_files(dir_noise)
# also load states files to recover noise indices
dir_states = os.path.join(self._dir_root_ac, "Features", "Mixed",
"States", set_name)
files_states = retrieve_files(dir_states)
# plot all categories if not given
filenames = [f for f in sorted(os.listdir(dir_mixed))]
if categories is None:
file_categories = [f.split("_")[0] for f in filenames]
categories = sorted(list(set(file_categories)))
# setup figure, subfigures
n_categories = len(categories)
n_variants = 1 + int(plot_clean) + int(plot_noise)
fig = plt.figure(
figsize=(6 * n_categories, 4 * n_variants),
constrained_layout=False,
)
gs = fig.add_gridspec(n_variants * 2, n_categories)
# loop through categories
for i, cat in enumerate(categories):
# get filename
fn = [f for f in filenames if f.split("_")[0] == cat][idx]
# load mixed file
file_mixed = os.path.join(dir_mixed, fn)
M = pd.read_csv(file_mixed, header=None).to_numpy()
# store spectra and titles
spectra = []
titles = []
if plot_clean:
# load clean file
file_clean = os.path.join(dir_clean, fn)
C = pd.read_csv(file_clean, header=None).to_numpy()
# add to lists
spectra.append(C)
titles.append("'%s': Clean Sound (%s)" %
(fn, self._feature["feature"]))
if plot_noise:
# get filename of states file belonging to noise file
fn_states = os.path.split([
f for f in files_states if fn.replace(".csv", "") in f
][0])[-1]
# get file idx, frame idx from states filename for noise file
idx_file = int(fn_states.split("_")[-2])
idx_frame = int(
fn_states.split("_")[-1].split(".")[0]) # omit .csv
# load noise file from idx_file
file_noise = files_noise[idx_file]
# get noise fragment from idx_frame
N = pd.read_csv(file_noise,
header=None).to_numpy()[:,
idx_frame:idx_frame +
M.shape[1]]
# add deltas to noise fragment
N = np.concatenate(
(N, librosa.feature.delta(N, mode="mirror")), axis=0)
# add to lists
spectra.append(N)
titles.append("MAV Noise (%s)" % self._feature["feature"])
# add mixed file to end of lists
spectra.append(M)
titles.append("'%s': Noisy Mix (%s)" %
(fn, self._feature["feature"]))
for j, Z in enumerate(spectra):
# plot spectrum
ax = fig.add_subplot(gs[2 * j, i])
ph.plot_spectrum(Z[:Z.shape[0] // 2], self._feature)
ax.set_title(titles[j])
# plot delta-spectrum
ax = fig.add_subplot(gs[2 * j + 1, i])
ph.plot_spectrum(Z[Z.shape[0] // 2:],
self._feature,
colormap="coolwarm")
def plot_spectra_clean(self, set_name, categories=None, idx=0):
"""Plot a selection of audio and spectra of a noise-free dataset.
!!! fix featuers
Keyword arguments:
set_name -- set to be plotted (e.g. 'Test'),
features -- iterable containing the features to be plotted
(default: all),
categories -- iterable containing the original categories
(airplane, engine, etc.) to be plotted (default: all),
idx -- index of the feature selected for plotting (default: 0).
Example usage:
plot_spectra_clean('Train Pitch Shift', ['airplane', 'wind'],
['Mel'])
This plots the first Melspectrum of the airplane and wind
categories in the Pitch Shift augmentation set.
This function does not verify whether the desired data exists, e.g. it
does not know whether 'Pitch Shift' augmentation is used or if
Melspectra were extracted.
"""
# list all audio files
dir_audio = os.path.join(self._dir_root, "Aircraft Classification",
"Audio", set_name)
files_audio = [
os.path.join(dir_audio, f) for f in sorted(os.listdir(dir_audio))
]
# plot all categories if not given
if categories is None:
file_categories = [
os.path.split(f)[-1].split("_")[0] for f in files_audio
]
categories = sorted(list(set(file_categories)))
# setup figure, subfigures
n_categories = len(categories)
fig = plt.figure(figsize=(6 * n_categories, 4),
constrained_layout=False)
gs = fig.add_gridspec(1, n_categories)
# loop through categories
for i, cat in enumerate(categories):
# load audio file
file_audio = [
f for f in files_audio
if os.path.split(f)[-1].split("_")[0] == cat
][idx]
fn_stub = os.path.split(file_audio)[-1].split(".")[0]
# prepare grid (plot audio half the size of features)
gs_in = gs[i].subgridspec(
3,
1,
hspace=0.0,
height_ratios=[1] + [2] * 2,
)
# plot audio
ax = fig.add_subplot(gs_in[0])
ph.plot_audio(file_audio, sr=self._feature["fft_sample_rate"])
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax.set_title(fn_stub)
# load feature
file_feature = os.path.join(
self._dir_root_ac,
"Features",
"Clean",
"Spectra",
set_name,
fn_stub + ".csv",
)
Z = pd.read_csv(file_feature, header=None).to_numpy()
# plot spectrum
ax = fig.add_subplot(gs_in[1])
ph.plot_spectrum(Z[:Z.shape[0] // 2], self._feature)
ax.get_xaxis().set_visible(False)
# only keep left y-axis visible
if i != 0:
ax.get_yaxis().set_visible(False)
# plot delta-spectrum
ax = fig.add_subplot(gs_in[2])
ph.plot_spectrum(Z[Z.shape[0] // 2:],
self._feature,
colormap="coolwarm")
ax.get_yaxis().set_visible(False)
def plot_predictions(self, df, categories=None, idx=0, plot_title="default"):
if categories is None:
categories = df["Category"].unique().tolist()
n_categories = len(categories)
n_features = 1 + int(self._feature["use_delta"])
if self._spectrum["feature"] == "Mfcc":
f_bins = self._spectrum["mfc_coefficients"]
else:
f_bins = self._spectrum["frequency_bins"]
# plot with correct aspect ratio
fig_width = 0.07 * n_categories * self._feature["segment_frames"]
fig_height = 0.07 * n_features * f_bins # keep evenly sized
fig = plt.figure(figsize=(fig_width, fig_height), constrained_layout=True)
gs = fig.add_gridspec(1, n_categories)
for i, cat in enumerate(categories):
gs_in = gs[0, i].subgridspec(n_features, 1)
# get file to be plotted
df_cat = df[df["Category"] == cat]
# load file
file_plot = df_cat["Filepath_Spectrum"].iloc[idx]
Z = pd.read_csv(file_plot, header=None).to_numpy()
# plot spectrum, add to grid
ax = fig.add_subplot(gs_in[0])
ph.plot_spectrum(Z[: Z.shape[0] // 2], self._spectrum)
# set title
if plot_title == "default":
if self._spectrum["feature"] == "Stft":
feat_name = "Spectrogram"
elif self._spectrum["feature"] == "Mel":
feat_name = "Mel spectrogram"
elif self._spectrum["feature"] == "Cqt":
feat_name = "Constant-Q spectrogram"
else:
feat_name = "MFCCs"
title = "%s (%d bins) of class '%s'" % (feat_name, f_bins, cat)
ax.set_title(title)
elif plot_title == "prediction":
fn = os.path.split(file_plot)[-1]
pred = df_cat["Predicted"].iloc[idx]
title = "'%s' (predicted: %.3f)" % (fn.split(".")[0], pred)
ax.set_title(title)
if self._feature["use_delta"]:
# plot delta-spectrum
ax = fig.add_subplot(gs_in[1])
ph.plot_spectrum(
Z[Z.shape[0] // 2 :], self._spectrum, colormap="coolwarm"
)
# if j > 0:
# ax.get_yaxis().set_visible(False)
# fig.tight_layout()
plt.show()
return fig