def test_symmetry(self): """Test that that accuracy between (A,B) and (B,A) is the same.""" sequence1 = [1] * 10 + [2] * 20 + [3] * 30 + [4] * 40 sequence2 = [1] * 10 + [2] * 20 + [3] * 30 + [4] * 40 random.shuffle(sequence1) random.shuffle(sequence2) accuracy1 = evals.compute_sequence_match_accuracy( sequence1, sequence2) accuracy2 = evals.compute_sequence_match_accuracy( sequence2, sequence1) self.assertEqual(accuracy1, accuracy2)
def test_different_num_unique_ids(self): """Test for two sequences with different number of unique ids.""" sequence1 = [1, 1] sequence2 = [1, 2] accuracy = evals.compute_sequence_match_accuracy( sequence1, sequence2) self.assertEqual(0.5, accuracy)
def test_equivalent_sequences(self): """Test for two sequences that are equivalent.""" sequence1 = [0, 0, 1, 2, 2] sequence2 = [3, 3, 4, 1, 1] accuracy = evals.compute_sequence_match_accuracy( sequence1, sequence2) self.assertEqual(1.0, accuracy)
def test_mismatched_sequences(self): """Test for two sequences that are different.""" sequence1 = [0, 0, 1, 2, 2] sequence2 = [3, 3, 4, 4, 1] accuracy = evals.compute_sequence_match_accuracy( sequence1, sequence2) self.assertEqual(0.8, accuracy)
def test(model_args, training_args, inference_args): """Test pipeline. test model --> output result Args: model_args: model configurations training_args: training configurations inference_args: inference configurations """ predicted_labels = [] test_record = [] test_data = np.load(os.getcwd() + '/speaker_diarization/model/testing_data.npz') test_sequences = test_data['test_sequence'] test_cluster_ids = test_data['test_cluster_id'] test_sequence_list = [ seq.astype(float) + 0.00001 for seq in test_sequences ] test_cluster_id_list = [ np.array(cid).astype(str) for cid in test_cluster_ids ] model = UISRNN(model_args) # testing model.load(SAVED_MODEL_NAME) for (test_sequence, test_cluster_id) in zip(test_sequence_list, test_cluster_id_list): predicted_label = model.predict(test_sequence, inference_args) predicted_labels.append(predicted_label) accuracy = evals.compute_sequence_match_accuracy( test_cluster_id, predicted_label) test_record.append((accuracy, len(test_cluster_id))) print('Ground truth labels:') print(test_cluster_id) print('Predicted labels:') print(predicted_label) print('-' * 80) output_string = diarization_utils.output_result(model_args, training_args, test_record) print('Finished diarization experiment') print(output_string)
def test_empty_sequences(self): """Test that empty sequences will raise error.""" with self.assertRaises(Exception): evals.compute_sequence_match_accuracy([], [])
def test_sequences_of_different_lengths(self): """Test that sequences of different lengths will raise error.""" sequence1 = [0, 0, 1, 2] sequence2 = [3, 3, 4, 4, 1] with self.assertRaises(Exception): evals.compute_sequence_match_accuracy(sequence1, sequence2)
def test_four_clusters(self): """Four clusters on vertices of a square.""" label_to_center = { 'A': np.array([0.0, 0.0]), 'B': np.array([0.0, 1.0]), 'C': np.array([1.0, 0.0]), 'D': np.array([1.0, 1.0]), } # generate training data train_cluster_id = ['A'] * 400 + ['B'] * 300 + ['C'] * 200 + ['D' ] * 100 random.shuffle(train_cluster_id) train_sequence = _generate_random_sequence(train_cluster_id, label_to_center, sigma=0.01) train_sequences = [ train_sequence[:100, :], train_sequence[100:300, :], train_sequence[300:600, :], train_sequence[600:, :] ] train_cluster_ids = [ train_cluster_id[:100], train_cluster_id[100:300], train_cluster_id[300:600], train_cluster_id[600:] ] # generate testing data test_cluster_id = ['A'] * 10 + ['B'] * 20 + ['C'] * 30 + ['D'] * 40 random.shuffle(test_cluster_id) test_sequence = _generate_random_sequence(test_cluster_id, label_to_center, sigma=0.01) # construct model model_args, training_args, inference_args = arguments.parse_arguments() model_args.rnn_depth = 2 model_args.rnn_hidden_size = 8 model_args.observation_dim = 2 model_args.verbosity = 3 training_args.learning_rate = 0.01 training_args.learning_rate_half_life = 50 training_args.train_iteration = 200 training_args.enforce_cluster_id_uniqueness = False inference_args.test_iteration = 2 model = UISRNN(model_args) # run training, and save the model model.fit(train_sequences, train_cluster_ids, training_args) temp_file_path = tempfile.mktemp() model.save(temp_file_path) # run testing predicted_label = model.predict(test_sequence, inference_args) # run evaluation model.logger.print( 3, 'Asserting the equivalence between' '\nGround truth: {}\nPredicted: {}'.format(test_cluster_id, predicted_label)) accuracy = evals.compute_sequence_match_accuracy( predicted_label, test_cluster_id) self.assertEqual(1.0, accuracy) # load new model loaded_model = UISRNN(model_args) loaded_model.load(temp_file_path) # run testing with loaded model predicted_label = loaded_model.predict(test_sequence, inference_args) # run evaluation with loaded model model.logger.print( 3, 'Asserting the equivalence between' '\nGround truth: {}\nPredicted: {}'.format(test_cluster_id, predicted_label)) accuracy = evals.compute_sequence_match_accuracy( predicted_label, test_cluster_id) self.assertEqual(1.0, accuracy) # keep training from loaded model on a subset of training data transition_bias_1 = model.transition_bias training_args.learning_rate = 0.001 training_args.train_iteration = 50 model.fit(train_sequence[:100, :], train_cluster_id[:100], training_args) transition_bias_2 = model.transition_bias self.assertNotAlmostEqual(transition_bias_1, transition_bias_2) model.logger.print( 3, 'Asserting transition_bias changed from {} to {}'.format( transition_bias_1, transition_bias_2)) # run evaluation model.logger.print( 3, 'Asserting the equivalence between' '\nGround truth: {}\nPredicted: {}'.format(test_cluster_id, predicted_label)) accuracy = evals.compute_sequence_match_accuracy( predicted_label, test_cluster_id) self.assertEqual(1.0, accuracy)