def evaluate(metrics,
model,
inputs,
targets,
onset_threshold=0.5,
frame_threshold=0.5,
save_path=None):
# NB: this can't be decorated with tf.function because of all the extract_notes functions not being pure TF code.
mel = audio_to_mel(inputs)
onset_pred, offset_pred, frame_pred, velocity_pred = model(mel,
training=False)
onset_labels, offset_labels, frame_labels, velocity_labels, path_labels = targets
# for key, loss in losses.items():
# metrics[key].append(loss.item()) # todo: add loss metrics
# We're working with batch size of 1, so remove the first index for everything.
onset_pred = tf.squeeze(onset_pred)
offset_pred = tf.squeeze(offset_pred)
frame_pred = tf.squeeze(frame_pred)
velocity_pred = tf.squeeze(velocity_pred)
onset_labels = tf.squeeze(onset_labels)
offset_labels = tf.squeeze(offset_labels)
frame_labels = tf.squeeze(frame_labels)
velocity_labels = tf.squeeze(velocity_labels)
path_labels = tf.squeeze(path_labels).numpy().decode("utf-8")
p_ref, i_ref, v_ref = extract_notes(onset_labels, frame_labels,
velocity_labels)
p_est, i_est, v_est = extract_notes(onset_pred, frame_pred, velocity_pred,
onset_threshold, frame_threshold)
t_ref, f_ref = notes_to_frames(p_ref, i_ref, frame_labels.shape)
t_est, f_est = notes_to_frames(p_est, i_est, frame_pred.shape)
scaling = HOP_LENGTH / SAMPLE_RATE
i_ref = (i_ref * scaling).reshape(-1, 2)
p_ref = np.array([midi_to_hz(MIN_MIDI + midi) for midi in p_ref])
i_est = (i_est * scaling).reshape(-1, 2)
p_est = np.array([midi_to_hz(MIN_MIDI + midi) for midi in p_est])
t_ref = t_ref.astype(np.float64) * scaling
f_ref = [
np.array([midi_to_hz(MIN_MIDI + midi) for midi in freqs])
for freqs in f_ref
]
t_est = t_est.astype(np.float64) * scaling
f_est = [
np.array([midi_to_hz(MIN_MIDI + midi) for midi in freqs])
for freqs in f_est
]
p, r, f, o = evaluate_notes(i_ref, p_ref, i_est, p_est, offset_ratio=None)
metrics['metric/note/precision'].append(p)
metrics['metric/note/recall'].append(r)
metrics['metric/note/f1'].append(f)
metrics['metric/note/overlap'].append(o)
p, r, f, o = evaluate_notes(i_ref, p_ref, i_est, p_est)
metrics['metric/note-with-offsets/precision'].append(p)
metrics['metric/note-with-offsets/recall'].append(r)
metrics['metric/note-with-offsets/f1'].append(f)
metrics['metric/note-with-offsets/overlap'].append(o)
p, r, f, o = evaluate_notes_with_velocity(i_ref,
p_ref,
v_ref,
i_est,
p_est,
v_est,
offset_ratio=None,
velocity_tolerance=0.1)
metrics['metric/note-with-velocity/precision'].append(p)
metrics['metric/note-with-velocity/recall'].append(r)
metrics['metric/note-with-velocity/f1'].append(f)
metrics['metric/note-with-velocity/overlap'].append(o)
p, r, f, o = evaluate_notes_with_velocity(i_ref,
p_ref,
v_ref,
i_est,
p_est,
v_est,
velocity_tolerance=0.1)
metrics['metric/note-with-offsets-and-velocity/precision'].append(p)
metrics['metric/note-with-offsets-and-velocity/recall'].append(r)
metrics['metric/note-with-offsets-and-velocity/f1'].append(f)
metrics['metric/note-with-offsets-and-velocity/overlap'].append(o)
frame_metrics = evaluate_frames(t_ref, f_ref, t_est, f_est)
metrics['metric/frame/f1'].append(
hmean(
[frame_metrics['Precision'] + eps, frame_metrics['Recall'] +
eps]) - eps)
for key, loss in frame_metrics.items():
metrics['metric/frame/' + key.lower().replace(' ', '_')].append(loss)
if save_path is not None:
os.makedirs(save_path, exist_ok=True)
label_path = os.path.join(save_path,
os.path.basename(path_labels) + '.label.png')
save_pianoroll(label_path, onset_labels, frame_labels)
pred_path = os.path.join(save_path,
os.path.basename(path_labels) + '.pred.png')
save_pianoroll(pred_path, onset_pred, frame_pred)
midi_path = os.path.join(save_path,
os.path.basename(path_labels) + '.pred.mid')
save_midi(midi_path, p_est, i_est, v_est)
return metrics
def evaluate(data, model, onset_threshold=0.5, frame_threshold=0.5, save_path=None):
metrics = defaultdict(list)
for label in data:
pred, losses = model.run_on_batch(label)
for key, loss in losses.items():
metrics[key].append(loss.item())
for key, value in pred.items():
value.squeeze_(0).relu_()
p_ref, i_ref, v_ref = extract_notes(label['onset'], label['frame'], label['velocity'])
p_est, i_est, v_est = extract_notes(pred['onset'], pred['frame'], pred['velocity'], onset_threshold, frame_threshold)
t_ref, f_ref = notes_to_frames(p_ref, i_ref, label['frame'].shape)
t_est, f_est = notes_to_frames(p_est, i_est, pred['frame'].shape)
scaling = HOP_LENGTH / SAMPLE_RATE
i_ref = (i_ref * scaling).reshape(-1, 2)
p_ref = np.array([midi_to_hz(MIN_MIDI + midi) for midi in p_ref])
i_est = (i_est * scaling).reshape(-1, 2)
p_est = np.array([midi_to_hz(MIN_MIDI + midi) for midi in p_est])
t_ref = t_ref.astype(np.float64) * scaling
f_ref = [np.array([midi_to_hz(MIN_MIDI + midi) for midi in freqs]) for freqs in f_ref]
t_est = t_est.astype(np.float64) * scaling
f_est = [np.array([midi_to_hz(MIN_MIDI + midi) for midi in freqs]) for freqs in f_est]
p, r, f, o = evaluate_notes(i_ref, p_ref, i_est, p_est, offset_ratio=None)
metrics['metric/note/precision'].append(p)
metrics['metric/note/recall'].append(r)
metrics['metric/note/f1'].append(f)
metrics['metric/note/overlap'].append(o)
p, r, f, o = evaluate_notes(i_ref, p_ref, i_est, p_est)
metrics['metric/note-with-offsets/precision'].append(p)
metrics['metric/note-with-offsets/recall'].append(r)
metrics['metric/note-with-offsets/f1'].append(f)
metrics['metric/note-with-offsets/overlap'].append(o)
p, r, f, o = evaluate_notes_with_velocity(i_ref, p_ref, v_ref, i_est, p_est, v_est,
offset_ratio=None, velocity_tolerance=0.1)
metrics['metric/note-with-velocity/precision'].append(p)
metrics['metric/note-with-velocity/recall'].append(r)
metrics['metric/note-with-velocity/f1'].append(f)
metrics['metric/note-with-velocity/overlap'].append(o)
p, r, f, o = evaluate_notes_with_velocity(i_ref, p_ref, v_ref, i_est, p_est, v_est, velocity_tolerance=0.1)
metrics['metric/note-with-offsets-and-velocity/precision'].append(p)
metrics['metric/note-with-offsets-and-velocity/recall'].append(r)
metrics['metric/note-with-offsets-and-velocity/f1'].append(f)
metrics['metric/note-with-offsets-and-velocity/overlap'].append(o)
frame_metrics = evaluate_frames(t_ref, f_ref, t_est, f_est)
metrics['metric/frame/f1'].append(hmean([frame_metrics['Precision'] + eps, frame_metrics['Recall'] + eps]) - eps)
for key, loss in frame_metrics.items():
metrics['metric/frame/' + key.lower().replace(' ', '_')].append(loss)
if save_path is not None:
os.makedirs(save_path, exist_ok=True)
label_path = os.path.join(save_path, os.path.basename(label['path']) + '.label.png')
save_pianoroll(label_path, label['onset'], label['frame'])
pred_path = os.path.join(save_path, os.path.basename(label['path']) + '.pred.png')
save_pianoroll(pred_path, pred['onset'], pred['frame'])
midi_path = os.path.join(save_path, os.path.basename(label['path']) + '.pred.mid')
save_midi(midi_path, p_est, i_est, v_est)
return metrics
def evaluate(data, model, logging_info, onset_threshold=0.5, frame_threshold=0.5, save_path=None):
metrics = defaultdict(list)
song_names = list()
for label in data:
song_names.append((label['path']).split("/")[-1:])
pred, losses = model.run_on_batch(label)
for key, loss in losses.items():
metrics[key].append(loss.item())
for key, value in pred.items():
value.squeeze_(0).relu_()
p_ref, i_ref, v_ref = extract_notes(label['onset'], label['frame'], label['velocity'])
p_est, i_est, v_est = extract_notes(pred['onset'], pred['frame'], pred['velocity'], onset_threshold, frame_threshold)
t_ref, f_ref = notes_to_frames(p_ref, i_ref, label['frame'].shape)
t_est, f_est = notes_to_frames(p_est, i_est, pred['frame'].shape)
scaling = HOP_LENGTH / SAMPLE_RATE
i_ref = (i_ref * scaling).reshape(-1, 2)
p_ref = np.array([midi_to_hz(MIN_MIDI + midi) for midi in p_ref])
i_est = (i_est * scaling).reshape(-1, 2)
p_est = np.array([midi_to_hz(MIN_MIDI + midi) for midi in p_est])
t_ref = t_ref.astype(np.float64) * scaling
f_ref = [np.array([midi_to_hz(MIN_MIDI + midi) for midi in freqs]) for freqs in f_ref]
t_est = t_est.astype(np.float64) * scaling
f_est = [np.array([midi_to_hz(MIN_MIDI + midi) for midi in freqs]) for freqs in f_est]
p, r, f, o = evaluate_notes(i_ref, p_ref, i_est, p_est, offset_ratio=None)
metrics['metric/note/precision'].append(p)
metrics['metric/note/recall'].append(r)
metrics['metric/note/f1'].append(f)
metrics['metric/note/overlap'].append(o)
p, r, f, o = evaluate_notes(i_ref, p_ref, i_est, p_est)
metrics['metric/note-with-offsets/precision'].append(p)
metrics['metric/note-with-offsets/recall'].append(r)
metrics['metric/note-with-offsets/f1'].append(f)
metrics['metric/note-with-offsets/overlap'].append(o)
p, r, f, o = evaluate_notes_with_velocity(i_ref, p_ref, v_ref, i_est, p_est, v_est,
offset_ratio=None, velocity_tolerance=0.1)
metrics['metric/note-with-velocity/precision'].append(p)
metrics['metric/note-with-velocity/recall'].append(r)
metrics['metric/note-with-velocity/f1'].append(f)
metrics['metric/note-with-velocity/overlap'].append(o)
p, r, f, o = evaluate_notes_with_velocity(i_ref, p_ref, v_ref, i_est, p_est, v_est, velocity_tolerance=0.1)
metrics['metric/note-with-offsets-and-velocity/precision'].append(p)
metrics['metric/note-with-offsets-and-velocity/recall'].append(r)
metrics['metric/note-with-offsets-and-velocity/f1'].append(f)
metrics['metric/note-with-offsets-and-velocity/overlap'].append(o)
frame_metrics = evaluate_frames(t_ref, f_ref, t_est, f_est)
metrics['metric/frame/f1'].append(hmean([frame_metrics['Precision'] + eps, frame_metrics['Recall'] + eps]) - eps)
for key, loss in frame_metrics.items():
metrics['metric/frame/' + key.lower().replace(' ', '_')].append(loss)
if save_path is not None:
os.makedirs(save_path, exist_ok=True)
label_path = os.path.join(save_path, os.path.basename(label['path']) + '.label.png')
save_pianoroll(label_path, label['onset'], label['frame'])
pred_path = os.path.join(save_path, os.path.basename(label['path']) + '.pred.png')
save_pianoroll(pred_path, pred['onset'], pred['frame'])
midi_path = os.path.join(save_path, os.path.basename(label['path']) + '.pred.mid')
save_midi(midi_path, p_est, i_est, v_est)
# Creating a table of results for each song, and sorting by note_with_offset F1 score
rename_dict = dict()
for key, _ in metrics.items():
if key.startswith('loss/'):
_, category = key.split('/')
rename_dict[key] = category + " loss"
if key.startswith('metric/'):
_, category, name = key.split('/')
rename_dict[key] = category + " " + name
model_file, dataset_name = logging_info
log_str = (model_file + "_" + dataset_name).replace("/", "-")
evaluation_by_song_df = pd.DataFrame.from_dict(metrics, orient='index').transpose()
evaluation_by_song_df.insert(0, "song_name", song_names)
evaluation_by_song_df["song_name"] = song_names
evaluation_by_song_df.rename(columns=rename_dict, inplace=True)
evaluation_by_song_df.sort_values("note-with-offsets f1", ascending=True)
evaluation_by_song_df.to_csv("./evaluations/new_evals" + log_str+ "_by_song.csv", index=False)
model_df = evaluation_by_song_df.mean()
model_df.to_csv("./evaluations/new_evals/new_evals" + log_str+ "_model.csv", index=False)
return metrics