class TestDataManagerTest(DuplicateTestCase): @overrides def setUp(self): super(TestDataManagerTest, self).setUp() self.write_duplicate_questions_train_file() self.write_duplicate_questions_test_file() self.data_manager = DataManager(PairFeature) self.data_manager.get_train_data_from_file([self.TRAIN_FILE]) def test_get_test_data_default(self): get_test_gen, test_size = self.data_manager.get_test_data_from_file( [self.TEST_FILE]) assert test_size == 3 test_gen = get_test_gen() inputs1, labels1 = test_gen.__next__() assert_allclose(inputs1[0], np.array([2, 1])) assert_allclose(inputs1[1], np.array([1, 0])) inputs2, labels2 = test_gen.__next__() assert_allclose(inputs2[0], np.array([4, 0])) assert_allclose(inputs2[1], np.array([5, 1])) inputs3, labels3 = test_gen.__next__() assert_allclose(inputs3[0], np.array([6, 0])) assert_allclose(inputs3[1], np.array([7, 0])) # Should raise a StopIteration with self.assertRaises(StopIteration): test_gen.__next__() # Test that we can make a new test generator new_test_gen = get_test_gen() # Verify that the new and old generator are not the same object assert new_test_gen != test_gen new_inputs1, new_labels1 = new_test_gen.__next__() assert_allclose(new_inputs1, inputs1) assert_allclose(new_labels1, labels1) new_inputs2, new_labels2 = new_test_gen.__next__() assert_allclose(new_inputs2, inputs2) assert_allclose(new_labels2, labels2) new_inputs3, new_labels3 = new_test_gen.__next__() assert_allclose(new_inputs3, inputs3) assert_allclose(new_labels3, labels3) # Should raise a StopIteration with self.assertRaises(StopIteration): new_test_gen.__next__() def test_get_test_data_default_character(self): get_test_gen, test_size = self.data_manager.get_test_data_from_file( [self.TEST_FILE], mode="character") test_gen = get_test_gen() assert test_size == 3 inputs1, labels = test_gen.__next__() assert_allclose( inputs1[0], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 10, 0, 0, 0], [6, 9, 2, 7, 8, 3, 5, 4, 9, 4, 1, 10]])) assert_allclose( inputs1[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 9, 4, 1, 11], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert len(labels) == 0 inputs2, labels = test_gen.__next__() assert_allclose( inputs2[0], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 12, 19, 17, 18], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose( inputs2[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 13, 0, 0, 0], [6, 9, 2, 7, 8, 3, 5, 4, 9, 4, 1, 12]])) assert len(labels) == 0 inputs3, labels = test_gen.__next__() assert_allclose( inputs3[0], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 14, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose( inputs3[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 15, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert len(labels) == 0 # Should raise a StopIteration with self.assertRaises(StopIteration): test_gen.__next__() def test_get_test_data_default_word_and_character(self): get_test_gen, test_size = self.data_manager.get_test_data_from_file( [self.TEST_FILE], mode="word+character") test_gen = get_test_gen() assert test_size == 3 inputs1, labels = test_gen.__next__() assert_allclose(inputs1[0], np.array([2, 1])) assert_allclose( inputs1[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 10, 0, 0, 0], [6, 9, 2, 7, 8, 3, 5, 4, 9, 4, 1, 10]])) assert_allclose(inputs1[2], np.array([1, 0])) assert_allclose( inputs1[3], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 9, 4, 1, 11], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert len(labels) == 0 inputs2, labels = test_gen.__next__() assert_allclose(inputs2[0], np.array([4, 0])) assert_allclose( inputs2[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 12, 19, 17, 18], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose(inputs2[2], np.array([5, 1])) assert_allclose( inputs2[3], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 13, 0, 0, 0], [6, 9, 2, 7, 8, 3, 5, 4, 9, 4, 1, 12]])) assert len(labels) == 0 inputs3, labels = test_gen.__next__() assert_allclose(inputs3[0], np.array([6, 0])) assert_allclose( inputs3[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 14, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose(inputs3[2], np.array([7, 0])) assert_allclose( inputs3[3], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 15, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert len(labels) == 0 # Should raise a StopIteration with self.assertRaises(StopIteration): test_gen.__next__() def test_get_test_data_pad_with_max_lens(self): get_test_gen, test_size = self.data_manager.get_test_data_from_file( [self.TEST_FILE], max_lengths={"num_sentence_words": 1}) test_gen = get_test_gen() assert test_size == 3 inputs, labels = test_gen.__next__() assert_allclose(inputs[0], np.array([2])) assert_allclose(inputs[1], np.array([1])) assert len(labels) == 0 inputs, labels = test_gen.__next__() assert_allclose(inputs[0], np.array([4])) assert_allclose(inputs[1], np.array([5])) assert len(labels) == 0 inputs, labels = test_gen.__next__() assert_allclose(inputs[0], np.array([6])) assert_allclose(inputs[1], np.array([7])) assert len(labels) == 0 # Should raise a StopIteration with self.assertRaises(StopIteration): test_gen.__next__() def test_get_test_data_with_max_features(self): get_test_gen, test_size = self.data_manager.get_test_data_from_file( [self.TEST_FILE], max_features=2) test_gen = get_test_gen() assert test_size == 2 inputs, labels = test_gen.__next__() assert_allclose(inputs[0], np.array([2, 1])) assert_allclose(inputs[1], np.array([1, 0])) assert len(labels) == 0 inputs, labels = test_gen.__next__() assert_allclose(inputs[0], np.array([4, 0])) assert_allclose(inputs[1], np.array([5, 1])) assert len(labels) == 0 # Should raise a StopIteration with self.assertRaises(StopIteration): test_gen.__next__() def test_get_test_data_errors(self): with self.assertRaises(ValueError): self.data_manager.get_test_data_from_file( [self.TEST_FILE], max_lengths={"num_sentence_words": 1}, pad=False) with self.assertRaises(ValueError): self.data_manager.get_test_data_from_file( [self.TEST_FILE], max_lengths={"some wrong key": 1}) def test_get_test_data_no_pad(self): get_test_gen, test_size = self.data_manager.get_test_data_from_file( [self.TEST_FILE], pad=False) test_gen = get_test_gen() assert test_size == 3 inputs, labels = test_gen.__next__() assert_allclose(inputs[0], np.array([2, 1, 2])) assert_allclose(inputs[1], np.array([1])) assert len(labels) == 0 inputs, labels = test_gen.__next__() assert_allclose(inputs[0], np.array([4])) assert_allclose(inputs[1], np.array([5, 1])) assert len(labels) == 0 inputs, labels = test_gen.__next__() assert_allclose(inputs[0], np.array([6])) assert_allclose(inputs[1], np.array([7])) assert len(labels) == 0 # Should raise a StopIteration with self.assertRaises(StopIteration): test_gen.__next__() def test_generate_test_batches(self): get_test_gen, test_size = self.data_manager.get_test_data_from_file( [self.TEST_FILE]) batch_gen = self.data_manager.get_batch_generator(get_test_gen, 2) new_batch_gen = DataManager.get_batch_generator(get_test_gen, 2) # Assert that the new generator is a different object # than the old generator. assert new_batch_gen != batch_gen assert test_size == 3 first_batch = batch_gen.__next__() new_first_batch = new_batch_gen.__next__() inputs, labels = first_batch new_inputs, new_labels = new_first_batch assert len(inputs) == 2 assert len(labels) == 0 # Ensure output matches ground truth assert_allclose(inputs[0], np.array([[2, 1], [4, 0]])) assert_allclose(inputs[1], np.array([[1, 0], [5, 1]])) # Ensure both generators produce same results. assert_allclose(inputs[0], new_inputs[0]) assert_allclose(inputs[1], new_inputs[1]) second_batch = batch_gen.__next__() new_second_batch = new_batch_gen.__next__() inputs, labels = second_batch new_inputs, new_labels = new_second_batch assert len(inputs) == 2 assert len(labels) == 0 # Ensure output matches ground truth assert_allclose(inputs[0], np.array([[6, 0]])) assert_allclose(inputs[1], np.array([[7, 0]])) # Ensure both generators produce same results. assert_allclose(inputs[0], new_inputs[0]) assert_allclose(inputs[1], new_inputs[1]) with self.assertRaises(StopIteration): batch_gen.__next__() new_batch_gen.__next__()
feature_type=QuoraFeature, pad=True, pickle_dir='../models/', max_lengths={"num_sentence_words": 30}) # In[ ]: data_manager = DataManager(dataset=quora_dataset) # In[ ]: get_train_data_gen, train_data_size = data_manager.get_train_data_from_file() # In[ ]: get_val_data_gen, val_data_size = data_manager.get_validation_data_from_file() # In[ ]: # embedding_manager = EmbeddingManager(quora_dataset.data_indexer, pickle_dir='../models/') # embedding_matrix = embedding_manager.get_embedding_matrix(300,'../data/quora/external/glove.6B.300d.txt') nlp = spacy.load('en_core_web_md') embedding_matrix = EmbeddingManager.get_spacy_embedding_matrix(nlp)