class EvaluatorTests(unittest.TestCase):
"""
Tests basic evaluation metric computation.
"""
def __init__(self, *args, **kwargs):
super(EvaluatorTests, self).__init__(*args, **kwargs)
self.test_instance = Evaluator()
def test_row_metrics(self):
"""
Tests correct computation of row-wise metrics.
:return: None.
"""
TestCase = namedtuple('Testcase', 'actual predicted precision recall')
cases = []
cases.append(
TestCase('percussion_openhat_30', 'percussion_openhat_30', 1, 1))
cases.append(
TestCase('percussion_openhat_30',
'percussion_openhat_30, percussion_rimshot_30', 0.5, 1))
cases.append(
TestCase('percussion_openhat_30, percussion_rimshot_30',
'percussion_openhat_30', 1, 0.5))
cases.append(
TestCase('percussion_rimshot_30', 'percussion_openhat_30', 0, 0))
for case in cases:
(precision, recall,
fscore) = self.test_instance.compute_row_accuracy(
case.predicted, case.actual)
self.assertTrue(precision == case.precision)
self.assertTrue(recall == case.recall)
return
def test_seq_metrics(self):
"""
Tests correct computation of sequence-based (average) metrics.
:return: None.
"""
TestCase = namedtuple('Testcase', 'actual predicted precision recall')
cases = []
cases.append(
TestCase(['percussion_openhat_30', 'percussion_rimshot_30'],
['percussion_openhat_30', 'percussion_rimshot_30'], 1, 1))
cases.append(
TestCase(['percussion_openhat_30', 'percussion_openhat_30'], [
'percussion_openhat_30, percussion_rimshot_30',
'percussion_openhat_30, percussion_rimshot_30'
], 0.5, 1))
cases.append(
TestCase([
'percussion_openhat_30, percussion_rimshot_30',
'percussion_openhat_30, percussion_rimshot_30'
], ['percussion_openhat_30', 'percussion_openhat_30'], 1, 0.5))
cases.append(
TestCase(['percussion_rimshot_30', 'percussion_rimshot_30'],
['percussion_openhat_30', 'percussion_openhat_30'], 0, 0))
for case in cases:
(precision, recall,
fscore) = self.test_instance.compute_seq_accuracy(
case.predicted, case.actual)
self.assertTrue(precision == case.precision)
self.assertTrue(recall == case.recall)
return
def test_error_metrics(self):
"""
Tests that computation of average error between vectors returns reasonable values.
:return: None.
"""
predicted = np.random.rand(10, 2)
actual = np.random.rand(10, 2)
error = self.test_instance.compute_average_error(predicted, actual)
self.assertTrue(error > 0)
return
def main():
# Documents used to train semantic encoder model
encoder_training_docs = "../resources/encoder_training_docs/full_1_measure_20k.txt"
model_params = {
# Encoder (doc2vec) settings:
'encoder_training_docs': encoder_training_docs,
'doc2vec_dm': 1,
'doc2vec_dm_mean': 1,
'doc2vec_epochs': 1,
'doc2vec_hs': 0,
'doc2vec_learning_rate_start': 0.025,
'doc2vec_learning_rate_end': 0.2,
'doc2vec_min_count': 5,
'doc2vec_negative': 0,
'doc2vec_vector_size': 5,
'doc2vec_window': 1,
# Sequence learning (Keras LSTM) settings:
'nn_features': ['bpm', 'measure', 'beat'],
'nn_batch_size': 100,
'nn_dense_activation_function': "linear",
'nn_dropout': 0.05,
'nn_epochs': 5,
'nn_hidden_neurons': 10,
'nn_layers': 10,
'nn_lstm_activation_function': "selu",
'nn_lstm_n_prev': 4
}
# Train encoder
encoder = Encoder(model_params)
docs = encoder.load_documents(model_params['encoder_training_docs'])
encoder.set_documents(docs)
encoder.train()
# Define note mapper for MIDI file loading
note_mapping_config_path = "../settings/map-to-group.json"
note_mapper = NoteMapper(note_mapping_config_path)
# Define training documents for sequence learning
training_docs = [
"/Users/taylorpeer/Projects/se-project/midi-embeddings/data/corpora/test/training"
] # TODO paths...
# Define evaluation documents for sequence learning
evaluation_docs = []
evaluation_docs.append(
"/Users/taylorpeer/Projects/se-project/midi-embeddings/data/corpora/test/test"
) # TODO paths...
# Load training MIDI files using MidiDataLoader
data_loader = MidiDataLoader(note_mapper,
params=model_params,
encoder=encoder)
training_data = data_loader.load_data_as_array(training_docs)
# Set fit_scaler=False to re-use scaler from training set
test_data = data_loader.load_data_as_array(evaluation_docs,
fit_scaler=False)
(x_test, y_test) = test_data
# Train sequence learning model
sequence_model = GenerativeSequenceLearner(model_params)
sequence_model.train(training_data)
# Apply trained model to test set
predicted = sequence_model.predict(x_test)
# Evaluate accuracy of model on test set
evaluator = Evaluator()
average_error = evaluator.compute_average_error(predicted, y_test)
# Un-scale predicted and actual values
scaler = data_loader.get_scaler()
predicted = scaler.inverse_transform(predicted)
y_test = scaler.inverse_transform(y_test)
# Convert predicted vectors to note sequence
predicted_notes = encoder.convert_feature_vectors_to_text(predicted)
# Convert actual vectors to note sequence
actual_notes = encoder.convert_feature_vectors_to_text(y_test)
# Compute accuracy by measuring precision/recall of predicted vs. actual notes at every timestamp of evaluation
(precision, recall,
f1) = evaluator.compute_seq_accuracy(predicted_notes, actual_notes)
# Remove doc2vec_docs params setting, since otherwise params can't be printed
model_params = dict((key, value) for key, value in model_params.items()
if key != 'doc2vec_docs')
print(str(model_params))
print("- precision: " + str(precision))
print("- recall: " + str(recall))
print("- f1: " + str(f1))
print("- average error: " + str(average_error))
print("---")
