def test_symmetry(self): sequence1 = [1] * 10 + [2] * 20 + [3] * 30 + [4] * 40 sequence2 = [1] * 10 + [2] * 20 + [3] * 30 + [4] * 40 random.shuffle(sequence1) random.shuffle(sequence2) accuracy1 = evals.compute_sequence_match_accuracy(sequence1, sequence2) accuracy2 = evals.compute_sequence_match_accuracy(sequence2, sequence1) self.assertEqual(accuracy1, accuracy2)
def test_four_clusters(self): """Four clusters on vertices of a square.""" label_to_center = { 'A': np.array([0.0, 0.0]), 'B': np.array([0.0, 1.0]), 'C': np.array([1.0, 0.0]), 'D': np.array([1.0, 1.0]), } # generate training data train_cluster_id = ['A'] * 400 + ['B'] * 300 + ['C'] * 200 + ['D'] * 100 random.shuffle(train_cluster_id) train_sequence = _generate_random_sequence( train_cluster_id, label_to_center, sigma=0.01) # generate testing data test_cluster_id = ['A'] * 10 + ['B'] * 20 + ['C'] * 30 + ['D'] * 40 random.shuffle(test_cluster_id) test_sequence = _generate_random_sequence( test_cluster_id, label_to_center, sigma=0.01) # construct model model_args, training_args, inference_args = arguments.parse_arguments() model_args.rnn_depth = 2 model_args.rnn_hidden_size = 8 model_args.observation_dim = 2 training_args.learning_rate = 0.01 training_args.learning_rate_half_life = 50 training_args.train_iteration = 200 inference_args.test_iteration = 2 model = uisrnn.UISRNN(model_args) # run training, and save the model model.fit(train_sequence, train_cluster_id, training_args) temp_file_path = tempfile.mktemp() model.save(temp_file_path) # run testing predicted_label = model.predict(test_sequence, inference_args) # run evaluation accuracy = evals.compute_sequence_match_accuracy( predicted_label, test_cluster_id) self.assertEqual(1.0, accuracy) # load new model loaded_model = uisrnn.UISRNN(model_args) loaded_model.load(temp_file_path) # run testing with loaded model predicted_label = loaded_model.predict(test_sequence, inference_args) # run evaluation with loaded model accuracy = evals.compute_sequence_match_accuracy( predicted_label, test_cluster_id) self.assertEqual(1.0, accuracy)
def diarization_experiment(model_args, training_args, inference_args): """Experiment pipeline. Load data --> train model --> test model --> output result Args: model_args: model configurations training_args: training configurations inference_args: inference configurations """ predicted_labels = [] test_record = [] train_data = np.load('./data/training_data.npz') test_data = np.load('./data/testing_data.npz') train_sequence = train_data['train_sequence'] train_cluster_id = train_data['train_cluster_id'] test_sequences = test_data['test_sequences'] test_cluster_ids = test_data['test_cluster_ids'] model = uisrnn.UISRNN(model_args) # training model.fit(train_sequence, train_cluster_id, training_args) model.save(SAVED_MODEL_NAME) # we can also skip training by calling: # model.load(SAVED_MODEL_NAME) # testing for (test_sequence, test_cluster_id) in zip(test_sequences, test_cluster_ids): predicted_label = model.predict(test_sequence, inference_args) predicted_labels.append(predicted_label) accuracy = evals.compute_sequence_match_accuracy( test_cluster_id, predicted_label) test_record.append((accuracy, len(test_cluster_id))) print('Ground truth labels:') print(test_cluster_id) print('Predicted labels:') print(predicted_label) print('-' * 80) output_string = utils.output_result(model_args, training_args, test_record) print('Finished diarization experiment') print(output_string)
def test_empty_sequences(self): with self.assertRaises(Exception): evals.compute_sequence_match_accuracy([], [])
def test_sequences_of_different_lengths(self): sequence1 = [0, 0, 1, 2] sequence2 = [3, 3, 4, 4, 1] with self.assertRaises(Exception): evals.compute_sequence_match_accuracy(sequence1, sequence2)
def test_equivalent_sequences(self): sequence1 = [0, 0, 1, 2, 2] sequence2 = [3, 3, 4, 1, 1] accuracy = evals.compute_sequence_match_accuracy( sequence1, sequence2) self.assertEqual(1.0, accuracy)
def test_mismatched_sequences(self): sequence1 = [0, 0, 1, 2, 2] sequence2 = [3, 3, 4, 4, 1] accuracy = evals.compute_sequence_match_accuracy( sequence1, sequence2) self.assertEqual(0.8, accuracy)
def test_different_num_unique_ids(self): sequence1 = [1, 1] sequence2 = [1, 2] accuracy = evals.compute_sequence_match_accuracy(sequence1, sequence2) self.assertEqual(0.5, accuracy)