def test_symmetry(self):
sequence1 = [1] * 10 + [2] * 20 + [3] * 30 + [4] * 40
sequence2 = [1] * 10 + [2] * 20 + [3] * 30 + [4] * 40
random.shuffle(sequence1)
random.shuffle(sequence2)
accuracy1 = evals.compute_sequence_match_accuracy(sequence1, sequence2)
accuracy2 = evals.compute_sequence_match_accuracy(sequence2, sequence1)
self.assertEqual(accuracy1, accuracy2)
def test_four_clusters(self):
"""Four clusters on vertices of a square."""
label_to_center = {
'A': np.array([0.0, 0.0]),
'B': np.array([0.0, 1.0]),
'C': np.array([1.0, 0.0]),
'D': np.array([1.0, 1.0]),
}
# generate training data
train_cluster_id = ['A'] * 400 + ['B'] * 300 + ['C'] * 200 + ['D'] * 100
random.shuffle(train_cluster_id)
train_sequence = _generate_random_sequence(
train_cluster_id, label_to_center, sigma=0.01)
# generate testing data
test_cluster_id = ['A'] * 10 + ['B'] * 20 + ['C'] * 30 + ['D'] * 40
random.shuffle(test_cluster_id)
test_sequence = _generate_random_sequence(
test_cluster_id, label_to_center, sigma=0.01)
# construct model
model_args, training_args, inference_args = arguments.parse_arguments()
model_args.rnn_depth = 2
model_args.rnn_hidden_size = 8
model_args.observation_dim = 2
training_args.learning_rate = 0.01
training_args.learning_rate_half_life = 50
training_args.train_iteration = 200
inference_args.test_iteration = 2
model = uisrnn.UISRNN(model_args)
# run training, and save the model
model.fit(train_sequence, train_cluster_id, training_args)
temp_file_path = tempfile.mktemp()
model.save(temp_file_path)
# run testing
predicted_label = model.predict(test_sequence, inference_args)
# run evaluation
accuracy = evals.compute_sequence_match_accuracy(
predicted_label, test_cluster_id)
self.assertEqual(1.0, accuracy)
# load new model
loaded_model = uisrnn.UISRNN(model_args)
loaded_model.load(temp_file_path)
# run testing with loaded model
predicted_label = loaded_model.predict(test_sequence, inference_args)
# run evaluation with loaded model
accuracy = evals.compute_sequence_match_accuracy(
predicted_label, test_cluster_id)
self.assertEqual(1.0, accuracy)
def diarization_experiment(model_args, training_args, inference_args):
"""Experiment pipeline.
Load data --> train model --> test model --> output result
Args:
model_args: model configurations
training_args: training configurations
inference_args: inference configurations
"""
predicted_labels = []
test_record = []
train_data = np.load('./data/training_data.npz')
test_data = np.load('./data/testing_data.npz')
train_sequence = train_data['train_sequence']
train_cluster_id = train_data['train_cluster_id']
test_sequences = test_data['test_sequences']
test_cluster_ids = test_data['test_cluster_ids']
model = uisrnn.UISRNN(model_args)
# training
model.fit(train_sequence, train_cluster_id, training_args)
model.save(SAVED_MODEL_NAME)
# we can also skip training by calling：
# model.load(SAVED_MODEL_NAME)
# testing
for (test_sequence, test_cluster_id) in zip(test_sequences,
test_cluster_ids):
predicted_label = model.predict(test_sequence, inference_args)
predicted_labels.append(predicted_label)
accuracy = evals.compute_sequence_match_accuracy(
test_cluster_id, predicted_label)
test_record.append((accuracy, len(test_cluster_id)))
print('Ground truth labels:')
print(test_cluster_id)
print('Predicted labels:')
print(predicted_label)
print('-' * 80)
output_string = utils.output_result(model_args, training_args, test_record)
print('Finished diarization experiment')
print(output_string)
def test_empty_sequences(self):
with self.assertRaises(Exception):
evals.compute_sequence_match_accuracy([], [])
def test_sequences_of_different_lengths(self):
sequence1 = [0, 0, 1, 2]
sequence2 = [3, 3, 4, 4, 1]
with self.assertRaises(Exception):
evals.compute_sequence_match_accuracy(sequence1, sequence2)
def test_equivalent_sequences(self):
sequence1 = [0, 0, 1, 2, 2]
sequence2 = [3, 3, 4, 1, 1]
accuracy = evals.compute_sequence_match_accuracy(
sequence1, sequence2)
self.assertEqual(1.0, accuracy)
def test_mismatched_sequences(self):
sequence1 = [0, 0, 1, 2, 2]
sequence2 = [3, 3, 4, 4, 1]
accuracy = evals.compute_sequence_match_accuracy(
sequence1, sequence2)
self.assertEqual(0.8, accuracy)
def test_different_num_unique_ids(self):
sequence1 = [1, 1]
sequence2 = [1, 2]
accuracy = evals.compute_sequence_match_accuracy(sequence1, sequence2)
self.assertEqual(0.5, accuracy)