def __init__(self, nr_of_representations: int, nr_of_samples: int,
chord_vocabulary: ChordVocabulary):
self.array = np.zeros((nr_of_representations, nr_of_samples),
dtype=int)
self._first_empty_row = 0
self._nr_of_representations = nr_of_representations
self._nr_of_samples = nr_of_samples
self._chord_alphabet = ChordAlphabet(chord_vocabulary)
def _read_tab_file_path(chords_from_tab_file_path: str, alphabet: ChordAlphabet) -> (int, np.array, np.array, np.array):
"""
Load chord information from chords_from_tab_file_path
:param chords_from_tab_file_path: File that contains chord information
:return: (nr_of_chords_in_tab, chord_ids, is_first_in_line, is_last_in_line)
"""
# Load .txt file consisting of: [line_nr, segment_nr, system_nr, chord_x, chord_str] (UntimedChordSequence)
untimed_chord_sequence = read_untimed_chord_sequence(chords_from_tab_file_path)
nr_of_chords_in_tab = len(untimed_chord_sequence.untimed_chord_sequence_item_items)
# If we found less than 5 chords, we will not use this tab
if nr_of_chords_in_tab < 5:
return nr_of_chords_in_tab, [], [], []
# Chord id's
line_nrs = [ucs_item.line_nr for ucs_item in untimed_chord_sequence.untimed_chord_sequence_item_items]
chord_ids = np.zeros(nr_of_chords_in_tab).astype(int)
for i in range(nr_of_chords_in_tab):
chord_ids[i] = alphabet.get_index_of_chord_in_alphabet(
Chord.from_harte_chord_string(untimed_chord_sequence.untimed_chord_sequence_item_items[i].chord_str))
# Array: is this chord first and/or last in its line?
is_first_in_line = np.zeros(nr_of_chords_in_tab).astype(int)
is_first_in_line[0] = 1
for i in range(1, nr_of_chords_in_tab):
if line_nrs[i] != line_nrs[i - 1]:
is_first_in_line[i] = 1
is_last_in_line = np.zeros(nr_of_chords_in_tab).astype(int)
is_last_in_line[-1] = 1
for i in range(nr_of_chords_in_tab - 1):
if line_nrs[i] != line_nrs[i + 1]:
is_last_in_line[i] = 1
return nr_of_chords_in_tab, chord_ids, is_first_in_line, is_last_in_line
def data_fuse_song_with_actual_best_midi_and_tab(
song: Song, chord_vocabulary: ChordVocabulary):
"""
Data fuse a song using all combinations of selection and combination methods, write the final labels to .lab files
:param song: The song on which we want to apply data fusion
:param chord_vocabulary: The chord vocabulary
"""
# Check if data fusion has already been calculated TODO: make this check more robust
if path.isfile(
filehandler.get_data_fusion_path(song.key, 'df', 'actual-best',
'CHF_2017')):
return
# Get list of audio lab files
audio_labs = song.full_mirex_chord_lab_paths
audio_labs['CHF_2017'] = song.full_chordify_chord_labs_path
# Sample every 10ms, so 100 samples per second
song_duration = song.duration
nr_of_samples = int(ceil(song_duration * 100))
# Turn the chords list (a list of (key, mode-str, chroma-list) tuples) into an chord_vocabulary (a list of strings)
alphabet = ChordAlphabet(chord_vocabulary)
selection_name = 'actual-best'
lab_list = [get_actual_best_tab_lab(song)]
# lab_list = [get_actual_best_midi_lab(song), get_actual_best_tab_lab(song)]
# Fill a numpy array with chord labels for each of the lab files
chord_matrix = np.zeros((len(lab_list) + 1, nr_of_samples), dtype=int)
for lab_nr in range(len(lab_list)):
load_lab_file_into_chord_matrix(lab_list[lab_nr], lab_nr, chord_matrix,
alphabet, nr_of_samples)
# Iterate over the audio types:
for audio_name, audio_lab in audio_labs.items():
if filehandler.file_exists(audio_lab):
# Add the lab file to our chord matrix
load_lab_file_into_chord_matrix(audio_lab, len(lab_list),
chord_matrix, alphabet,
nr_of_samples)
final_labels_data_fusion = _data_fusion_chord_label_combination(
chord_matrix, nr_of_samples, alphabet)
_write_final_labels(
final_labels_data_fusion,
filehandler.get_data_fusion_path(song.key, 'df',
selection_name, audio_name),
alphabet)
class ChordMatrix:
def __init__(self, nr_of_representations: int, nr_of_samples: int,
chord_vocabulary: ChordVocabulary):
self.array = np.zeros((nr_of_representations, nr_of_samples),
dtype=int)
self._first_empty_row = 0
self._nr_of_representations = nr_of_representations
self._nr_of_samples = nr_of_samples
self._chord_alphabet = ChordAlphabet(chord_vocabulary)
def append_lab_file(self, lab_path: str):
if self._first_empty_row < self._nr_of_representations:
self._load_lab_file_into_chord_matrix(lab_path,
self._first_empty_row)
self._first_empty_row += 1
else:
raise IndexError(
'Index out of bounds; the chord matrix seems to be full already.'
)
def _load_lab_file_into_chord_matrix(self, lab_path: str, i: int):
"""
Load a chord file (in Harte's chord annotation format) into a chord matrix.
:param lab_path: Path to the .lab file with chord annotation/estimation
:param i: Index to the row in the chord_matrix to fill
"""
with open(lab_path, 'r') as read_file:
# Read chord annotation from file
chord_annotation = read_file.readlines()
chord_annotation = [x.rstrip().split() for x in chord_annotation]
for y in chord_annotation:
# Parse start and end time, retrieve index of chord
start_time, end_time = float(y[0]), float(y[1])
chord = Chord.from_harte_chord_string(y[2])
chord_label_alphabet_index = self._chord_alphabet.get_index_of_chord_in_alphabet(
chord)
# Add chord index to each entry in the chord_matrix that is between start and end time
for s in range(
int(start_time * 100),
min(int(end_time * 100), self._nr_of_samples - 1)):
self.array[i, s] = chord_label_alphabet_index
def train(chord_vocabulary: ChordVocabulary,
train_songs: Dict[int, Song]) -> HMMParameters:
"""
Train the HMM parameters on training_set for the given chords_list vocabulary
:param chord_vocabulary: List of chords in our vocabulary
:param train_songs: Set of songs for training
:return: HMM Parameters
"""
# Convert the vocabulary to a ChordAlphabet
alphabet = ChordAlphabet(chord_vocabulary)
alphabet_size = len(alphabet.alphabet_list)
# Initialize chord_beat_matrix_per_chord: a list with |chord_vocabulary| x |beats| list for each chord
chroma_beat_matrix_per_chord = [[] for _ in alphabet.alphabet_list]
# Initialize transition_matrix and init_matrix
trans = np.ones((alphabet_size, alphabet_size))
init = np.ones(alphabet_size)
# Iterate over the songs; fill chroma_beat_matrix_per_chord, init_matrix and transition_matrix
for train_song_key, train_song in train_songs.items():
train_song.audio_features_path = filehandler.get_full_audio_features_path(
train_song_key)
if train_song.audio_features_path != '':
# We have audio features and labels for this song; load them (otherwise ignore the song)
features = np.load(train_song.audio_features_path)
chord_index_list = []
# Iterate over the beats, fill chroma_beat_matrix_per_chord and chord_index_list
for frame_index in range(features.shape[0]):
chroma = features[frame_index, 1:13].astype(float)
chord_index = alphabet.get_index_of_chord_in_alphabet(
Chord.from_harte_chord_string(features[frame_index, 13]))
chord_index_list.append(chord_index)
chroma_beat_matrix_per_chord[chord_index].append(chroma)
# Add first chord to init_matrix
init[chord_index_list[0]] += 1
# Add each chord transition to transition_matrix
for i in range(0, len(chord_index_list) - 1):
trans[chord_index_list[i], chord_index_list[i + 1]] += 1
# Normalize transition and init matrices
init = init / sum(init)
trans = np.array([trans[i] / sum(trans[i]) for i in range(alphabet_size)])
# Calculate mean and covariance matrices
obs_mu = np.zeros((alphabet_size, 12))
obs_sigma = np.zeros((alphabet_size, 12, 12))
for i in range(alphabet_size):
chroma_beat_matrix_per_chord[i] = np.array(
chroma_beat_matrix_per_chord[i]).T
obs_mu[i] = np.mean(chroma_beat_matrix_per_chord[i], axis=1)
obs_sigma[i] = np.cov(chroma_beat_matrix_per_chord[i], ddof=0)
# Calculate additional values so we can calculate the emission probability more easily
twelve_log_two_pi = 12 * np.log(2 * np.pi)
log_det_sigma = np.zeros(alphabet_size)
sigma_inverse = np.zeros(obs_sigma.shape)
for i in range(alphabet_size):
log_det_sigma[i] = np.log(np.linalg.det(obs_sigma[i]))
sigma_inverse[i] = np.mat(np.linalg.pinv(obs_sigma[i]))
return HMMParameters(alphabet=alphabet,
trans=trans,
init=init,
obs_mu=obs_mu,
obs_sigma=obs_sigma,
log_det_sigma=log_det_sigma,
sigma_inverse=sigma_inverse,
twelve_log_two_pi=twelve_log_two_pi,
trained_on_keys=list(train_songs.keys()))
def export_result_image(song: Song,
chords_vocabulary: ChordVocabulary,
midi: bool = True,
tab: bool = True,
audio: str = 'CHF_2017',
df: bool = True):
"""
Export visualisation to a png file.
:param song: Song for which we want to export the visualisation
:param chords_vocabulary: Chord vocabulary
:param midi: Show MIDI files?
:param tab: Show Tab files?
:param audio: Audio ACE method
:param df: Show all DF results?
"""
if filehandler.file_exists(
filehandler.get_lab_visualisation_path(song, audio)):
return song.title + " was already visualised for the ACE method " + audio + "."
nr_of_samples = int(ceil(song.duration * 100))
alphabet = ChordAlphabet(chords_vocabulary)
# Select labs based on parameter setting
label_data = [{
'name': 'Ground truth',
'index': 0,
'lab_path': song.full_ground_truth_chord_labs_path,
'csr': 1.0,
'ovs': 1.0,
'uns': 1.0,
'seg': 1.0
}]
i = 1
best_indices = [] # For expected best MIDI and tab
if midi:
duplicate_midis = filehandler.find_duplicate_midis(song)
best_midi_name, best_segmentation = data_fusion.get_expected_best_midi(
song)
full_midi_paths = song.full_midi_paths
full_midi_paths.sort()
for full_midi_path in full_midi_paths:
midi_name = filehandler.get_file_name_from_full_path(
full_midi_path)
for segmentation_method in ['bar', 'beat']:
full_midi_chords_path = filehandler.get_full_midi_chord_labs_path(
midi_name, segmentation_method)
if filehandler.file_exists(full_midi_chords_path) \
and midi_name not in duplicate_midis:
# Evaluate song
csr, ovs, uns, seg = evaluate(
song.full_ground_truth_chord_labs_path,
full_midi_chords_path)
# Save evaluation values to label_data
label_data.append({
'name':
'MIDI ' + midi_name + ' | ' + segmentation_method,
'index': i,
'lab_path': full_midi_chords_path,
'csr': csr,
'ovs': ovs,
'uns': uns,
'seg': seg
})
# Check if this is the expected best MIDI & segmentation method for this song
if midi_name == best_midi_name and segmentation_method == best_segmentation:
best_indices.append(i)
i += 1
if tab:
best_tab = data_fusion.get_expected_best_tab_lab(song)
for tab_counter, full_tab_path in enumerate(song.full_tab_paths, 1):
tab_chord_labs_path = filehandler.get_full_tab_chord_labs_path(
full_tab_path)
if filehandler.file_exists(tab_chord_labs_path):
# Evaluate song
csr, ovs, uns, seg = evaluate(
song.full_ground_truth_chord_labs_path,
tab_chord_labs_path)
# Save evaluation values to label_data
label_data.append({
'name': 'Tab ' + str(tab_counter),
'index': i,
'lab_path': tab_chord_labs_path,
'csr': csr,
'ovs': ovs,
'uns': uns,
'seg': seg
})
if tab_chord_labs_path == best_tab:
best_indices.append(i)
i += 1
if df:
csr, ovs, uns, seg = evaluate(
song.full_ground_truth_chord_labs_path,
filehandler.get_full_mirex_chord_labs_path(song, audio))
label_data.append({
'name':
audio,
'index':
i,
'lab_path':
filehandler.get_full_mirex_chord_labs_path(song, audio),
'csr':
csr,
'ovs':
ovs,
'uns':
uns,
'seg':
seg
})
for selection_name in 'all', 'best':
for combination_name in 'rnd', 'mv', 'df':
df_lab_path = filehandler.get_data_fusion_path(
song.key, combination_name, selection_name, audio)
csr, ovs, uns, seg = evaluate(
song.full_ground_truth_chord_labs_path, df_lab_path)
label_data.append({
'name':
audio + '-' + combination_name.upper() + '-' +
selection_name.upper(),
'index':
i,
'lab_path':
df_lab_path,
'csr':
csr,
'ovs':
ovs,
'uns':
uns,
'seg':
seg
})
# Fill a numpy array with chord labels for each of the lab files
chord_matrix = np.zeros((len(label_data), nr_of_samples), dtype=int)
for lab_nr in range(len(label_data)):
data_fusion.load_lab_file_into_chord_matrix(
label_data[lab_nr]['lab_path'], lab_nr, chord_matrix, alphabet,
nr_of_samples)
all_chords = [chord_matrix[x] for x in range(len(label_data))]
# Find names
names = [label_dict['name'] for label_dict in label_data]
# Find results
results = ['CSR OvS UnS Seg']
for label_dict in label_data[1:]:
results.append(' '.join([
str(round(label_dict[measure], 2)).ljust(4, '0')
for measure in ['csr', 'ovs', 'uns', 'seg']
]))
# Show result
plt1 = _show_chord_sequences(song, all_chords, best_indices, names,
results, alphabet)
plt1.savefig(filehandler.get_lab_visualisation_path(song, audio),
bbox_inches="tight",
pad_inches=0)
return song.title + " was visualised for the ACE method " + audio + "."
def data_fuse_song(song: Song, chord_vocabulary: ChordVocabulary):
"""
Data fuse a song using all combinations of selection and combination methods, write the final labels to .lab files
:param song: The song on which we want to apply data fusion
:param chord_vocabulary: The chord vocabulary
"""
# Check if data fusion has already been calculated TODO: make this check more robust
if path.isfile(
filehandler.get_data_fusion_path(song.key, 'df', 'besttab',
'CHF_2017')):
return
# Get list of symbolic lab files (all / expected best)
# well_aligned_midis = get_well_aligned_midis(song)
# all_symbolic_lab_paths = \
# [filehandler.get_full_midi_chord_labs_path(wam, 'bar') for wam in well_aligned_midis] + \
# [filehandler.get_full_midi_chord_labs_path(wam, 'beat') for wam in well_aligned_midis] + \
# [filehandler.get_full_tab_chord_labs_path(t) for t in song.full_tab_paths]
# expected_best_symbolic_lab_paths = []
# if well_aligned_midis:
# expected_best_symbolic_lab_paths.append(
# filehandler.get_full_midi_chord_labs_path(*get_expected_best_midi(song)))
# if [filehandler.get_full_tab_chord_labs_path(t) for t in song.full_tab_paths]:
# expected_best_symbolic_lab_paths.append(
# filehandler.get_full_tab_chord_labs_path(get_expected_best_tab_lab(song)))
well_aligned_midis = get_well_aligned_midis(song)
all_midi_bar_lab_paths = [
filehandler.get_full_midi_chord_labs_path(wam, 'bar')
for wam in well_aligned_midis if filehandler.file_exists(
filehandler.get_full_midi_chord_labs_path(wam, 'bar'))
]
all_midi_beat_lab_paths = [
filehandler.get_full_midi_chord_labs_path(wam, 'beat')
for wam in well_aligned_midis if filehandler.file_exists(
filehandler.get_full_midi_chord_labs_path(wam, 'beat'))
]
all_midi_lab_paths = all_midi_bar_lab_paths + all_midi_beat_lab_paths
all_tab_lab_paths = [
filehandler.get_full_tab_chord_labs_path(t)
for t in song.full_tab_paths
if filehandler.file_exists(filehandler.get_full_tab_chord_labs_path(t))
]
all_audio_lab_paths = {
**song.full_mirex_chord_lab_paths,
**{
'CHF_2017': song.full_chordify_chord_labs_path
}
}
# expected_best_symbolic_lab_paths = []
# if well_aligned_midis:
# expected_best_symbolic_lab_paths.append(
# filehandler.get_full_midi_chord_labs_path(*get_expected_best_midi(song)))
# if [filehandler.get_full_tab_chord_labs_path(t) for t in song.full_tab_paths]:
# expected_best_symbolic_lab_paths.append(
# filehandler.get_full_tab_chord_labs_path(get_expected_best_tab_lab(song)))
expected_best_midi_lab_paths = []
if well_aligned_midis:
expected_best_midi_lab_paths.append(
filehandler.get_full_midi_chord_labs_path(
*get_expected_best_midi(song)))
expected_best_tab_lab_paths = []
if [
filehandler.get_full_tab_chord_labs_path(t)
for t in song.full_tab_paths
]:
expected_best_tab_lab_paths.append(
filehandler.get_full_tab_chord_labs_path(
get_expected_best_tab_lab(song)))
all_symbolic_lab_paths = all_midi_lab_paths + all_tab_lab_paths
expected_best_symbolic_lab_paths = expected_best_midi_lab_paths + expected_best_tab_lab_paths
# # Remove non-existing files (e.g. tab files in which too little chords were observed)
# all_symbolic_lab_paths = [lab for lab in all_symbolic_lab_paths if filehandler.file_exists(lab)]
# expected_best_symbolic_lab_paths = [lab for lab in expected_best_symbolic_lab_paths if filehandler.file_exists(lab)]
# Get list of audio lab files
# audio_labs = song.full_mirex_chord_lab_paths
# audio_labs['CHF_2017'] = song.full_chordify_chord_labs_path
# Sample every 10ms, so 100 samples per second
song_duration = song.duration
nr_of_samples = int(ceil(song_duration * 100))
# Turn the chords list (a list of (key, mode-str, chroma-list) tuples) into an chord_vocabulary (a list of strings)
alphabet = ChordAlphabet(chord_vocabulary)
selection_dict = {
'all': all_symbolic_lab_paths,
'best': expected_best_symbolic_lab_paths,
'allmidi': all_midi_lab_paths,
'bestmidi': expected_best_midi_lab_paths,
'alltab': all_tab_lab_paths,
'besttab': expected_best_tab_lab_paths
}
# Iterate over the two types of selection (all / best)
# for lab_list_i in [0, 1]:
# lab_list = [all_symbolic_lab_paths, expected_best_symbolic_lab_paths,
# all_midi_lab_paths, expected_best_midi_lab_paths,
# all_tab_lab_paths, expected_best_tab_lab_paths][lab_list_i]
# lab_list = [i for i in lab_list if i != '']
# selection_name = ['all', 'best', 'allmidi', 'bestmidi', 'alltab', 'besttab'][lab_list_i]
for selection_name, lab_list in selection_dict.items():
lab_list = [i for i in lab_list if i != '']
# Fill a numpy array with chord labels for each of the lab files
chord_matrix = np.zeros((len(lab_list) + 1, nr_of_samples), dtype=int)
for lab_nr in range(len(lab_list)):
load_lab_file_into_chord_matrix(lab_list[lab_nr], lab_nr,
chord_matrix, alphabet,
nr_of_samples)
# Iterate over the audio types:
for audio_name, audio_lab in all_audio_lab_paths.items():
if filehandler.file_exists(audio_lab):
if any([
not filehandler.file_exists(
filehandler.get_data_fusion_path(
song.key, df_type_str, selection_name,
audio_name))
for df_type_str in ['rnd', 'mv', 'df']
]):
# Add the lab file to our chord matrix
load_lab_file_into_chord_matrix(audio_lab, len(lab_list),
chord_matrix, alphabet,
nr_of_samples)
# Iterate over the three combination types; calculate labels and write them:
if not filehandler.file_exists(
filehandler.get_data_fusion_path(
song.key, 'rnd', selection_name, audio_name)):
final_labels_random = _random_chord_label_combination(
chord_matrix, nr_of_samples)
_write_final_labels(
final_labels_random,
filehandler.get_data_fusion_path(
song.key, 'rnd', selection_name, audio_name),
alphabet)
if not filehandler.file_exists(
filehandler.get_data_fusion_path(
song.key, 'mv', selection_name, audio_name)):
final_labels_majority = _majority_vote_chord_label_combination(
chord_matrix, nr_of_samples, alphabet)
_write_final_labels(
final_labels_majority,
filehandler.get_data_fusion_path(
song.key, 'mv', selection_name, audio_name),
alphabet)
if not filehandler.file_exists(
filehandler.get_data_fusion_path(
song.key, 'df', selection_name, audio_name)):
final_labels_data_fusion = _data_fusion_chord_label_combination(
chord_matrix, nr_of_samples, alphabet)
_write_final_labels(
final_labels_data_fusion,
filehandler.get_data_fusion_path(
song.key, 'df', selection_name, audio_name),
alphabet)