def test_symmetry(self):
"""Test that that accuracy between (A,B) and (B,A) is the same."""
sequence1 = [1] * 10 + [2] * 20 + [3] * 30 + [4] * 40
sequence2 = [1] * 10 + [2] * 20 + [3] * 30 + [4] * 40
random.shuffle(sequence1)
random.shuffle(sequence2)
accuracy1 = evals.compute_sequence_match_accuracy(
sequence1, sequence2)
accuracy2 = evals.compute_sequence_match_accuracy(
sequence2, sequence1)
self.assertEqual(accuracy1, accuracy2)
def test_different_num_unique_ids(self):
"""Test for two sequences with different number of unique ids."""
sequence1 = [1, 1]
sequence2 = [1, 2]
accuracy = evals.compute_sequence_match_accuracy(
sequence1, sequence2)
self.assertEqual(0.5, accuracy)
def test_equivalent_sequences(self):
"""Test for two sequences that are equivalent."""
sequence1 = [0, 0, 1, 2, 2]
sequence2 = [3, 3, 4, 1, 1]
accuracy = evals.compute_sequence_match_accuracy(
sequence1, sequence2)
self.assertEqual(1.0, accuracy)
def test_mismatched_sequences(self):
"""Test for two sequences that are different."""
sequence1 = [0, 0, 1, 2, 2]
sequence2 = [3, 3, 4, 4, 1]
accuracy = evals.compute_sequence_match_accuracy(
sequence1, sequence2)
self.assertEqual(0.8, accuracy)
def test(model_args, training_args, inference_args):
"""Test pipeline.
test model --> output result
Args:
model_args: model configurations
training_args: training configurations
inference_args: inference configurations
"""
predicted_labels = []
test_record = []
test_data = np.load(os.getcwd() +
'/speaker_diarization/model/testing_data.npz')
test_sequences = test_data['test_sequence']
test_cluster_ids = test_data['test_cluster_id']
test_sequence_list = [
seq.astype(float) + 0.00001 for seq in test_sequences
]
test_cluster_id_list = [
np.array(cid).astype(str) for cid in test_cluster_ids
]
model = UISRNN(model_args)
# testing
model.load(SAVED_MODEL_NAME)
for (test_sequence, test_cluster_id) in zip(test_sequence_list,
test_cluster_id_list):
predicted_label = model.predict(test_sequence, inference_args)
predicted_labels.append(predicted_label)
accuracy = evals.compute_sequence_match_accuracy(
test_cluster_id, predicted_label)
test_record.append((accuracy, len(test_cluster_id)))
print('Ground truth labels:')
print(test_cluster_id)
print('Predicted labels:')
print(predicted_label)
print('-' * 80)
output_string = diarization_utils.output_result(model_args, training_args,
test_record)
print('Finished diarization experiment')
print(output_string)
def test_empty_sequences(self):
"""Test that empty sequences will raise error."""
with self.assertRaises(Exception):
evals.compute_sequence_match_accuracy([], [])
def test_sequences_of_different_lengths(self):
"""Test that sequences of different lengths will raise error."""
sequence1 = [0, 0, 1, 2]
sequence2 = [3, 3, 4, 4, 1]
with self.assertRaises(Exception):
evals.compute_sequence_match_accuracy(sequence1, sequence2)
def test_four_clusters(self):
"""Four clusters on vertices of a square."""
label_to_center = {
'A': np.array([0.0, 0.0]),
'B': np.array([0.0, 1.0]),
'C': np.array([1.0, 0.0]),
'D': np.array([1.0, 1.0]),
}
# generate training data
train_cluster_id = ['A'] * 400 + ['B'] * 300 + ['C'] * 200 + ['D'
] * 100
random.shuffle(train_cluster_id)
train_sequence = _generate_random_sequence(train_cluster_id,
label_to_center,
sigma=0.01)
train_sequences = [
train_sequence[:100, :], train_sequence[100:300, :],
train_sequence[300:600, :], train_sequence[600:, :]
]
train_cluster_ids = [
train_cluster_id[:100], train_cluster_id[100:300],
train_cluster_id[300:600], train_cluster_id[600:]
]
# generate testing data
test_cluster_id = ['A'] * 10 + ['B'] * 20 + ['C'] * 30 + ['D'] * 40
random.shuffle(test_cluster_id)
test_sequence = _generate_random_sequence(test_cluster_id,
label_to_center,
sigma=0.01)
# construct model
model_args, training_args, inference_args = arguments.parse_arguments()
model_args.rnn_depth = 2
model_args.rnn_hidden_size = 8
model_args.observation_dim = 2
model_args.verbosity = 3
training_args.learning_rate = 0.01
training_args.learning_rate_half_life = 50
training_args.train_iteration = 200
training_args.enforce_cluster_id_uniqueness = False
inference_args.test_iteration = 2
model = UISRNN(model_args)
# run training, and save the model
model.fit(train_sequences, train_cluster_ids, training_args)
temp_file_path = tempfile.mktemp()
model.save(temp_file_path)
# run testing
predicted_label = model.predict(test_sequence, inference_args)
# run evaluation
model.logger.print(
3, 'Asserting the equivalence between'
'\nGround truth: {}\nPredicted: {}'.format(test_cluster_id,
predicted_label))
accuracy = evals.compute_sequence_match_accuracy(
predicted_label, test_cluster_id)
self.assertEqual(1.0, accuracy)
# load new model
loaded_model = UISRNN(model_args)
loaded_model.load(temp_file_path)
# run testing with loaded model
predicted_label = loaded_model.predict(test_sequence, inference_args)
# run evaluation with loaded model
model.logger.print(
3, 'Asserting the equivalence between'
'\nGround truth: {}\nPredicted: {}'.format(test_cluster_id,
predicted_label))
accuracy = evals.compute_sequence_match_accuracy(
predicted_label, test_cluster_id)
self.assertEqual(1.0, accuracy)
# keep training from loaded model on a subset of training data
transition_bias_1 = model.transition_bias
training_args.learning_rate = 0.001
training_args.train_iteration = 50
model.fit(train_sequence[:100, :], train_cluster_id[:100],
training_args)
transition_bias_2 = model.transition_bias
self.assertNotAlmostEqual(transition_bias_1, transition_bias_2)
model.logger.print(
3, 'Asserting transition_bias changed from {} to {}'.format(
transition_bias_1, transition_bias_2))
# run evaluation
model.logger.print(
3, 'Asserting the equivalence between'
'\nGround truth: {}\nPredicted: {}'.format(test_cluster_id,
predicted_label))
accuracy = evals.compute_sequence_match_accuracy(
predicted_label, test_cluster_id)
self.assertEqual(1.0, accuracy)