def _note_metrics(labels, predictions):
"""A pyfunc that wraps a call to precision_recall_f1_overlap."""
est_sequence = pianoroll_to_note_sequence(
predictions,
frames_per_second=data.hparams_frames_per_second(hparams),
min_duration_ms=hparams.min_duration_ms)
ref_sequence = pianoroll_to_note_sequence(
labels,
frames_per_second=data.hparams_frames_per_second(hparams),
min_duration_ms=hparams.min_duration_ms)
est_intervals, est_pitches = sequence_to_valued_intervals(
est_sequence, hparams.min_duration_ms)
ref_intervals, ref_pitches = sequence_to_valued_intervals(
ref_sequence, hparams.min_duration_ms)
if est_intervals.size == 0 or ref_intervals.size == 0:
return 0., 0., 0.
note_precision, note_recall, note_f1, _ = precision_recall_f1_overlap(
ref_intervals,
pretty_midi.note_number_to_hz(ref_pitches),
est_intervals,
pretty_midi.note_number_to_hz(est_pitches),
offset_ratio=offset_ratio)
return note_precision, note_recall, note_f1
def _note_metrics(labels, predictions):
"""A pyfunc that wraps a call to precision_recall_f1_overlap."""
est_sequence = pianoroll_to_note_sequence(
predictions,
frames_per_second=data.hparams_frames_per_second(hparams),
min_duration_ms=hparams.min_duration_ms)
ref_sequence = pianoroll_to_note_sequence(
labels,
frames_per_second=data.hparams_frames_per_second(hparams),
min_duration_ms=hparams.min_duration_ms)
est_intervals, est_pitches = sequence_to_valued_intervals(
est_sequence, hparams.min_duration_ms)
ref_intervals, ref_pitches = sequence_to_valued_intervals(
ref_sequence, hparams.min_duration_ms)
if est_intervals.size == 0 or ref_intervals.size == 0:
return 0., 0., 0.
note_precision, note_recall, note_f1, _ = precision_recall_f1_overlap(
ref_intervals,
pretty_midi.note_number_to_hz(ref_pitches),
est_intervals,
pretty_midi.note_number_to_hz(est_pitches),
offset_ratio=offset_ratio)
return note_precision, note_recall, note_f1
def evaluation_note(ans_list, pred_list, output_dir, filename,
onset_tolerance=0.1, offset_ratio=0.2,
string=None, mode='w', verbose=False,
separator=' ', poly_mask=None, extension=''):
if poly_mask:
poly_mask = np.loadtxt(poly_mask)
ans_list = remove_poly_notes(ans_list, poly_mask)
pred_list = remove_poly_notes(pred_list, poly_mask)
ref_intervals, ref_pitches, est_intervals, est_pitches = fit_mir_eval_transcription(ans_list, pred_list)
result = []
# result with onset, pitch, offset on
note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(ref_intervals, ref_pitches,
est_intervals, est_pitches,
onset_tolerance=onset_tolerance,
offset_ratio=offset_ratio)
result += [round(note_p, 5)]+[round(note_r, 5)]+[round(note_f, 5)]+['']
# result with onset, pitch on
note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(ref_intervals, ref_pitches,
est_intervals, est_pitches,
onset_tolerance=onset_tolerance,
offset_ratio=None)
result += [round(note_p, 5)]+[round(note_r, 5)]+[round(note_f, 5)]+['']
# result with onset on
onset_f, onset_p, onset_r = f_measure(ref_intervals[:,0], est_intervals[:,0], window=onset_tolerance)
result += [round(onset_p, 5)]+[round(onset_r, 5)]+[round(onset_f, 5)]+['']
if string: result += [string]
if not os.path.exists(output_dir): os.makedirs(output_dir)
fh = open(output_dir+os.sep+filename+'.note.eval'+extension, mode)
w = csv.writer(fh, delimiter = ',')
if mode == 'w':
w.writerow(['C_On_P_Off','','','','C_On_P','','','','C_On','','','','Stage'])
w.writerow(result)
fh.close()
print result
def get_notewise_f_measure(df, gt_df):
"""
Get the notewise F-measure of a dataframe, given a ground truth dataframe.
Parameters
----------
df : pd.DataFrame
The dataframe whose notewise F-measure to return.
gt_df : pd.DataFrame
The ground truth dataframe.
time_increment : int
The length of a frame to use for quantization when performing
evaluations.
Returns
-------
f_measure : float
The notewise f_measure of the given dataframe, using mir_eval.
"""
gt_pitches = np.array(gt_df["pitch"])
gt_times = np.vstack((gt_df["onset"], gt_df["onset"] + gt_df["dur"])).T
pitches = np.array(df["pitch"])
times = np.vstack((df["onset"], df["onset"] + df["dur"])).T
_, _, fm, _ = precision_recall_f1_overlap(
gt_times,
gt_pitches,
times,
pitches,
onset_tolerance=50,
pitch_tolerance=0,
offset_ratio=None,
)
return fm
def evaluation_esn(ans_list,
pred_list,
output_dir,
filename,
onset_tolerance=0.1,
offset_ratio=None,
poly_mask=None,
string=None,
mode='w',
extension=''):
def append_result(data, type_str, P, R, F, TP, FP, FN):
data.append([
type_str,
round(P, 4), ' (' + str(TP) + '/' + str(TP + FP) + ')',
round(R, 4), ' (' + str(TP) + '/' + str(TP + FN) + ')',
round(F, 4)
])
def combine_evals(res_dict, tech_list):
result_lists = [res_dict[t] for t in tech_list]
TP, FP, FN = 0, 0, 0
TP = sum([res[3] for res in result_lists])
FP = sum([res[4] for res in result_lists])
FN = sum([res[5] for res in result_lists])
P = TP / float(TP + FP) if (TP != 0 or FP != 0) else 0
R = TP / float(TP + FN) if (TP != 0 or FN != 0) else 0
F = 2 * P * R / float(P + R) if (P != 0 or R != 0) else 0
return P, R, F, TP, FP, FN
if poly_mask:
poly_mask = np.loadtxt(poly_mask)
ans_list = remove_poly_esn(ans_list, poly_mask)
pred_list = remove_poly_esn(pred_list, poly_mask)
data = []
if string: data.append([string])
ref_intervals, ref_pitches, est_intervals, est_pitches = fit_mir_eval_transcription(
ans_list, pred_list)
note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(
ref_intervals,
ref_pitches,
est_intervals,
est_pitches,
onset_tolerance=onset_tolerance,
offset_ratio=offset_ratio)
data.append([
'C_On_P_Off',
round(note_p, 4), '',
round(note_r, 4), '',
round(note_f, 4)
])
eval_dict = {
T_NORMAL: [],
T_PREBEND: [],
T_BEND: [],
T_RELEASE: [],
T_PULL: [],
T_HAMMER: [],
T_SLIDE: [],
T_SLIDE_IN: [],
T_SLIDE_OUT: [],
T_VIBRATO: []
}
for tech in eval_dict:
eval_dict[tech] = list(
calculate_esn_f_measure(ans_list,
pred_list,
tech=tech,
onset_tolerance=onset_tolerance))
append_result(data, T_STR_DICT[tech], *(eval_dict[tech]))
P, R, F, TP, FP, FN = calculate_esn_f_measure(
ans_list, pred_list, tech=None, onset_tolerance=onset_tolerance)
append_result(data, 'All', P, R, F, TP, FP, FN)
data.append(['---------------------------------------'])
all_bend_res = combine_evals(eval_dict, [T_PREBEND, T_BEND, T_RELEASE])
append_result(data, T_STR_DICT[T_BEND], *all_bend_res)
append_result(data, T_STR_DICT[T_PULL], *(eval_dict[T_PULL]))
append_result(data, T_STR_DICT[T_HAMMER], *(eval_dict[T_HAMMER]))
all_slide_res = combine_evals(eval_dict,
[T_SLIDE, T_SLIDE_IN, T_SLIDE_OUT])
append_result(data, T_STR_DICT[T_SLIDE], *all_slide_res)
append_result(data, T_STR_DICT[T_VIBRATO], *(eval_dict[T_VIBRATO]))
data.append([''])
# write results
fh = open(output_dir + os.sep + filename + '.esn.eval' + extension, mode)
w = csv.writer(fh, delimiter=',')
if mode == 'w':
w.writerow(['', 'Precision', '', 'Recall', '', 'F-measure'])
for r in data:
w.writerow(r)
print r
fh.close()
def evaluation_note(ans_list,
pred_list,
output_dir,
filename,
onset_tolerance=0.1,
offset_ratio=0.2,
string=None,
mode='w',
verbose=False,
separator=' ',
poly_mask=None,
extension=''):
if poly_mask:
poly_mask = np.loadtxt(poly_mask)
ans_list = remove_poly_notes(ans_list, poly_mask)
pred_list = remove_poly_notes(pred_list, poly_mask)
ref_intervals, ref_pitches, est_intervals, est_pitches = fit_mir_eval_transcription(
ans_list, pred_list)
result = []
# result with onset, pitch, offset on
note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(
ref_intervals,
ref_pitches,
est_intervals,
est_pitches,
onset_tolerance=onset_tolerance,
offset_ratio=offset_ratio)
result += [round(note_p, 5)] + [round(note_r, 5)] + [round(note_f, 5)
] + ['']
# result with onset, pitch on
note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(
ref_intervals,
ref_pitches,
est_intervals,
est_pitches,
onset_tolerance=onset_tolerance,
offset_ratio=None)
result += [round(note_p, 5)] + [round(note_r, 5)] + [round(note_f, 5)
] + ['']
# result with onset on
onset_f, onset_p, onset_r = f_measure(ref_intervals[:, 0],
est_intervals[:, 0],
window=onset_tolerance)
result += [round(onset_p, 5)] + [round(onset_r, 5)] + [round(onset_f, 5)
] + ['']
if string: result += [string]
if not os.path.exists(output_dir): os.makedirs(output_dir)
fh = open(output_dir + os.sep + filename + '.note.eval' + extension, mode)
w = csv.writer(fh, delimiter=',')
if mode == 'w':
w.writerow([
'C_On_P_Off', '', '', '', 'C_On_P', '', '', '', 'C_On', '', '', '',
'Stage'
])
w.writerow(result)
fh.close()
print result
def evaluation_esn( ans_list, pred_list, output_dir, filename,
onset_tolerance=0.1, offset_ratio=None,
poly_mask=None, string=None, mode='w',
extension=''):
def append_result(data, type_str, P, R, F, TP, FP, FN):
data.append([type_str, round(P, 4) ,' ('+str(TP)+'/'+str(TP+FP)+')', round(R, 4), ' ('+str(TP)+'/'+str(TP+FN)+')', round(F, 4)])
def combine_evals(res_dict, tech_list):
result_lists = [res_dict[t] for t in tech_list]
TP, FP, FN = 0, 0, 0
TP = sum([res[3] for res in result_lists])
FP = sum([res[4] for res in result_lists])
FN = sum([res[5] for res in result_lists])
P = TP/float(TP+FP) if (TP != 0 or FP != 0) else 0
R = TP/float(TP+FN) if (TP != 0 or FN != 0) else 0
F = 2*P*R/float(P+R) if (P != 0 or R != 0) else 0
return P, R, F, TP, FP, FN
if poly_mask:
poly_mask = np.loadtxt(poly_mask)
ans_list = remove_poly_esn(ans_list, poly_mask)
pred_list = remove_poly_esn(pred_list, poly_mask)
data = []
if string: data.append([string])
ref_intervals, ref_pitches, est_intervals, est_pitches = fit_mir_eval_transcription(ans_list, pred_list)
note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(ref_intervals, ref_pitches,
est_intervals, est_pitches,
onset_tolerance=onset_tolerance,
offset_ratio=offset_ratio)
data.append(['C_On_P_Off', round(note_p, 4) ,'', round(note_r, 4), '', round(note_f, 4)])
eval_dict = {T_NORMAL:[], T_PREBEND:[], T_BEND:[],
T_RELEASE:[], T_PULL:[], T_HAMMER:[],
T_SLIDE:[], T_SLIDE_IN:[], T_SLIDE_OUT:[], T_VIBRATO:[]}
for tech in eval_dict:
eval_dict[tech] = list(calculate_esn_f_measure(ans_list, pred_list, tech=tech, onset_tolerance=onset_tolerance))
append_result(data, T_STR_DICT[tech], *(eval_dict[tech]))
P, R, F, TP, FP, FN = calculate_esn_f_measure(ans_list, pred_list, tech=None, onset_tolerance=onset_tolerance)
append_result(data, 'All', P, R, F, TP, FP, FN)
data.append(['---------------------------------------'])
all_bend_res = combine_evals(eval_dict, [T_PREBEND, T_BEND, T_RELEASE])
append_result(data, T_STR_DICT[T_BEND], *all_bend_res)
append_result(data, T_STR_DICT[T_PULL], *(eval_dict[T_PULL]))
append_result(data, T_STR_DICT[T_HAMMER], *(eval_dict[T_HAMMER]))
all_slide_res = combine_evals(eval_dict, [T_SLIDE, T_SLIDE_IN, T_SLIDE_OUT])
append_result(data, T_STR_DICT[T_SLIDE], *all_slide_res)
append_result(data, T_STR_DICT[T_VIBRATO], *(eval_dict[T_VIBRATO]))
data.append([''])
# write results
fh = open(output_dir+os.sep+filename+'.esn.eval'+extension,mode)
w = csv.writer(fh, delimiter = ',')
if mode=='w': w.writerow(['', 'Precision', '', 'Recall', '', 'F-measure'])
for r in data:
w.writerow(r)
print r
fh.close()