def _note_metrics(labels, predictions): """A pyfunc that wraps a call to precision_recall_f1_overlap.""" est_sequence = pianoroll_to_note_sequence( predictions, frames_per_second=data.hparams_frames_per_second(hparams), min_duration_ms=hparams.min_duration_ms) ref_sequence = pianoroll_to_note_sequence( labels, frames_per_second=data.hparams_frames_per_second(hparams), min_duration_ms=hparams.min_duration_ms) est_intervals, est_pitches = sequence_to_valued_intervals( est_sequence, hparams.min_duration_ms) ref_intervals, ref_pitches = sequence_to_valued_intervals( ref_sequence, hparams.min_duration_ms) if est_intervals.size == 0 or ref_intervals.size == 0: return 0., 0., 0. note_precision, note_recall, note_f1, _ = precision_recall_f1_overlap( ref_intervals, pretty_midi.note_number_to_hz(ref_pitches), est_intervals, pretty_midi.note_number_to_hz(est_pitches), offset_ratio=offset_ratio) return note_precision, note_recall, note_f1
def evaluation_note(ans_list, pred_list, output_dir, filename, onset_tolerance=0.1, offset_ratio=0.2, string=None, mode='w', verbose=False, separator=' ', poly_mask=None, extension=''): if poly_mask: poly_mask = np.loadtxt(poly_mask) ans_list = remove_poly_notes(ans_list, poly_mask) pred_list = remove_poly_notes(pred_list, poly_mask) ref_intervals, ref_pitches, est_intervals, est_pitches = fit_mir_eval_transcription(ans_list, pred_list) result = [] # result with onset, pitch, offset on note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(ref_intervals, ref_pitches, est_intervals, est_pitches, onset_tolerance=onset_tolerance, offset_ratio=offset_ratio) result += [round(note_p, 5)]+[round(note_r, 5)]+[round(note_f, 5)]+[''] # result with onset, pitch on note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(ref_intervals, ref_pitches, est_intervals, est_pitches, onset_tolerance=onset_tolerance, offset_ratio=None) result += [round(note_p, 5)]+[round(note_r, 5)]+[round(note_f, 5)]+[''] # result with onset on onset_f, onset_p, onset_r = f_measure(ref_intervals[:,0], est_intervals[:,0], window=onset_tolerance) result += [round(onset_p, 5)]+[round(onset_r, 5)]+[round(onset_f, 5)]+[''] if string: result += [string] if not os.path.exists(output_dir): os.makedirs(output_dir) fh = open(output_dir+os.sep+filename+'.note.eval'+extension, mode) w = csv.writer(fh, delimiter = ',') if mode == 'w': w.writerow(['C_On_P_Off','','','','C_On_P','','','','C_On','','','','Stage']) w.writerow(result) fh.close() print result
def get_notewise_f_measure(df, gt_df): """ Get the notewise F-measure of a dataframe, given a ground truth dataframe. Parameters ---------- df : pd.DataFrame The dataframe whose notewise F-measure to return. gt_df : pd.DataFrame The ground truth dataframe. time_increment : int The length of a frame to use for quantization when performing evaluations. Returns ------- f_measure : float The notewise f_measure of the given dataframe, using mir_eval. """ gt_pitches = np.array(gt_df["pitch"]) gt_times = np.vstack((gt_df["onset"], gt_df["onset"] + gt_df["dur"])).T pitches = np.array(df["pitch"]) times = np.vstack((df["onset"], df["onset"] + df["dur"])).T _, _, fm, _ = precision_recall_f1_overlap( gt_times, gt_pitches, times, pitches, onset_tolerance=50, pitch_tolerance=0, offset_ratio=None, ) return fm
def evaluation_esn(ans_list, pred_list, output_dir, filename, onset_tolerance=0.1, offset_ratio=None, poly_mask=None, string=None, mode='w', extension=''): def append_result(data, type_str, P, R, F, TP, FP, FN): data.append([ type_str, round(P, 4), ' (' + str(TP) + '/' + str(TP + FP) + ')', round(R, 4), ' (' + str(TP) + '/' + str(TP + FN) + ')', round(F, 4) ]) def combine_evals(res_dict, tech_list): result_lists = [res_dict[t] for t in tech_list] TP, FP, FN = 0, 0, 0 TP = sum([res[3] for res in result_lists]) FP = sum([res[4] for res in result_lists]) FN = sum([res[5] for res in result_lists]) P = TP / float(TP + FP) if (TP != 0 or FP != 0) else 0 R = TP / float(TP + FN) if (TP != 0 or FN != 0) else 0 F = 2 * P * R / float(P + R) if (P != 0 or R != 0) else 0 return P, R, F, TP, FP, FN if poly_mask: poly_mask = np.loadtxt(poly_mask) ans_list = remove_poly_esn(ans_list, poly_mask) pred_list = remove_poly_esn(pred_list, poly_mask) data = [] if string: data.append([string]) ref_intervals, ref_pitches, est_intervals, est_pitches = fit_mir_eval_transcription( ans_list, pred_list) note_p, note_r, note_f, avg_or = precision_recall_f1_overlap( ref_intervals, ref_pitches, est_intervals, est_pitches, onset_tolerance=onset_tolerance, offset_ratio=offset_ratio) data.append([ 'C_On_P_Off', round(note_p, 4), '', round(note_r, 4), '', round(note_f, 4) ]) eval_dict = { T_NORMAL: [], T_PREBEND: [], T_BEND: [], T_RELEASE: [], T_PULL: [], T_HAMMER: [], T_SLIDE: [], T_SLIDE_IN: [], T_SLIDE_OUT: [], T_VIBRATO: [] } for tech in eval_dict: eval_dict[tech] = list( calculate_esn_f_measure(ans_list, pred_list, tech=tech, onset_tolerance=onset_tolerance)) append_result(data, T_STR_DICT[tech], *(eval_dict[tech])) P, R, F, TP, FP, FN = calculate_esn_f_measure( ans_list, pred_list, tech=None, onset_tolerance=onset_tolerance) append_result(data, 'All', P, R, F, TP, FP, FN) data.append(['---------------------------------------']) all_bend_res = combine_evals(eval_dict, [T_PREBEND, T_BEND, T_RELEASE]) append_result(data, T_STR_DICT[T_BEND], *all_bend_res) append_result(data, T_STR_DICT[T_PULL], *(eval_dict[T_PULL])) append_result(data, T_STR_DICT[T_HAMMER], *(eval_dict[T_HAMMER])) all_slide_res = combine_evals(eval_dict, [T_SLIDE, T_SLIDE_IN, T_SLIDE_OUT]) append_result(data, T_STR_DICT[T_SLIDE], *all_slide_res) append_result(data, T_STR_DICT[T_VIBRATO], *(eval_dict[T_VIBRATO])) data.append(['']) # write results fh = open(output_dir + os.sep + filename + '.esn.eval' + extension, mode) w = csv.writer(fh, delimiter=',') if mode == 'w': w.writerow(['', 'Precision', '', 'Recall', '', 'F-measure']) for r in data: w.writerow(r) print r fh.close()
def evaluation_note(ans_list, pred_list, output_dir, filename, onset_tolerance=0.1, offset_ratio=0.2, string=None, mode='w', verbose=False, separator=' ', poly_mask=None, extension=''): if poly_mask: poly_mask = np.loadtxt(poly_mask) ans_list = remove_poly_notes(ans_list, poly_mask) pred_list = remove_poly_notes(pred_list, poly_mask) ref_intervals, ref_pitches, est_intervals, est_pitches = fit_mir_eval_transcription( ans_list, pred_list) result = [] # result with onset, pitch, offset on note_p, note_r, note_f, avg_or = precision_recall_f1_overlap( ref_intervals, ref_pitches, est_intervals, est_pitches, onset_tolerance=onset_tolerance, offset_ratio=offset_ratio) result += [round(note_p, 5)] + [round(note_r, 5)] + [round(note_f, 5) ] + [''] # result with onset, pitch on note_p, note_r, note_f, avg_or = precision_recall_f1_overlap( ref_intervals, ref_pitches, est_intervals, est_pitches, onset_tolerance=onset_tolerance, offset_ratio=None) result += [round(note_p, 5)] + [round(note_r, 5)] + [round(note_f, 5) ] + [''] # result with onset on onset_f, onset_p, onset_r = f_measure(ref_intervals[:, 0], est_intervals[:, 0], window=onset_tolerance) result += [round(onset_p, 5)] + [round(onset_r, 5)] + [round(onset_f, 5) ] + [''] if string: result += [string] if not os.path.exists(output_dir): os.makedirs(output_dir) fh = open(output_dir + os.sep + filename + '.note.eval' + extension, mode) w = csv.writer(fh, delimiter=',') if mode == 'w': w.writerow([ 'C_On_P_Off', '', '', '', 'C_On_P', '', '', '', 'C_On', '', '', '', 'Stage' ]) w.writerow(result) fh.close() print result
def evaluation_esn( ans_list, pred_list, output_dir, filename, onset_tolerance=0.1, offset_ratio=None, poly_mask=None, string=None, mode='w', extension=''): def append_result(data, type_str, P, R, F, TP, FP, FN): data.append([type_str, round(P, 4) ,' ('+str(TP)+'/'+str(TP+FP)+')', round(R, 4), ' ('+str(TP)+'/'+str(TP+FN)+')', round(F, 4)]) def combine_evals(res_dict, tech_list): result_lists = [res_dict[t] for t in tech_list] TP, FP, FN = 0, 0, 0 TP = sum([res[3] for res in result_lists]) FP = sum([res[4] for res in result_lists]) FN = sum([res[5] for res in result_lists]) P = TP/float(TP+FP) if (TP != 0 or FP != 0) else 0 R = TP/float(TP+FN) if (TP != 0 or FN != 0) else 0 F = 2*P*R/float(P+R) if (P != 0 or R != 0) else 0 return P, R, F, TP, FP, FN if poly_mask: poly_mask = np.loadtxt(poly_mask) ans_list = remove_poly_esn(ans_list, poly_mask) pred_list = remove_poly_esn(pred_list, poly_mask) data = [] if string: data.append([string]) ref_intervals, ref_pitches, est_intervals, est_pitches = fit_mir_eval_transcription(ans_list, pred_list) note_p, note_r, note_f, avg_or = precision_recall_f1_overlap(ref_intervals, ref_pitches, est_intervals, est_pitches, onset_tolerance=onset_tolerance, offset_ratio=offset_ratio) data.append(['C_On_P_Off', round(note_p, 4) ,'', round(note_r, 4), '', round(note_f, 4)]) eval_dict = {T_NORMAL:[], T_PREBEND:[], T_BEND:[], T_RELEASE:[], T_PULL:[], T_HAMMER:[], T_SLIDE:[], T_SLIDE_IN:[], T_SLIDE_OUT:[], T_VIBRATO:[]} for tech in eval_dict: eval_dict[tech] = list(calculate_esn_f_measure(ans_list, pred_list, tech=tech, onset_tolerance=onset_tolerance)) append_result(data, T_STR_DICT[tech], *(eval_dict[tech])) P, R, F, TP, FP, FN = calculate_esn_f_measure(ans_list, pred_list, tech=None, onset_tolerance=onset_tolerance) append_result(data, 'All', P, R, F, TP, FP, FN) data.append(['---------------------------------------']) all_bend_res = combine_evals(eval_dict, [T_PREBEND, T_BEND, T_RELEASE]) append_result(data, T_STR_DICT[T_BEND], *all_bend_res) append_result(data, T_STR_DICT[T_PULL], *(eval_dict[T_PULL])) append_result(data, T_STR_DICT[T_HAMMER], *(eval_dict[T_HAMMER])) all_slide_res = combine_evals(eval_dict, [T_SLIDE, T_SLIDE_IN, T_SLIDE_OUT]) append_result(data, T_STR_DICT[T_SLIDE], *all_slide_res) append_result(data, T_STR_DICT[T_VIBRATO], *(eval_dict[T_VIBRATO])) data.append(['']) # write results fh = open(output_dir+os.sep+filename+'.esn.eval'+extension,mode) w = csv.writer(fh, delimiter = ',') if mode=='w': w.writerow(['', 'Precision', '', 'Recall', '', 'F-measure']) for r in data: w.writerow(r) print r fh.close()