class TestDataManagerValidation(DuplicateTestCase): @overrides def setUp(self): super(TestDataManagerValidation, self).setUp() self.write_duplicate_questions_train_file() self.write_duplicate_questions_validation_file() self.data_manager = DataManager(PairFeature) self.data_manager.get_train_data_from_file([self.TRAIN_FILE]) def test_get_validation_data_default(self): get_val_gen, val_size = self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE]) assert val_size == 3 val_gen = get_val_gen() inputs1, labels1 = val_gen.__next__() assert_allclose(inputs1[0], np.array([2, 0])) assert_allclose(inputs1[1], np.array([3, 1])) assert_allclose(labels1[0], np.array([1, 0])) inputs2, labels2 = val_gen.__next__() assert_allclose(inputs2[0], np.array([1, 0])) assert_allclose(inputs2[1], np.array([1, 0])) assert_allclose(labels2[0], np.array([0, 1])) inputs3, labels3 = val_gen.__next__() assert_allclose(inputs3[0], np.array([7, 0])) assert_allclose(inputs3[1], np.array([8, 1])) assert_allclose(labels3[0], np.array([1, 0])) # Should raise a StopIteration with self.assertRaises(StopIteration): val_gen.__next__() # Test that we can make a new val generator new_val_gen = get_val_gen() # Verify that the new and old generator are not the same object assert new_val_gen != val_gen new_inputs1, new_labels1 = new_val_gen.__next__() assert_allclose(new_inputs1, inputs1) assert_allclose(new_labels1, labels1) new_inputs2, new_labels2 = new_val_gen.__next__() assert_allclose(new_inputs2, inputs2) assert_allclose(new_labels2, labels2) new_inputs3, new_labels3 = new_val_gen.__next__() assert_allclose(new_inputs3, inputs3) assert_allclose(new_labels3, labels3) # Should raise a StopIteration with self.assertRaises(StopIteration): new_val_gen.__next__() def test_get_validation_data_default_character(self): get_val_gen, val_size = self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE], mode="character") assert val_size == 3 val_gen = get_val_gen() inputs1, labels1 = val_gen.__next__() assert_allclose( inputs1[0], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 10, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose( inputs1[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 11, 0, 0, 0], [6, 9, 2, 7, 8, 3, 5, 4, 1, 0, 0, 0]])) assert_allclose(labels1[0], np.array([1, 0])) inputs2, labels2 = val_gen.__next__() assert_allclose( inputs2[0], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose( inputs2[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 10, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose(labels2[0], np.array([0, 1])) inputs3, labels3 = val_gen.__next__() assert_allclose( inputs3[0], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 15, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose( inputs3[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 16, 0, 0, 0], [6, 9, 2, 7, 8, 3, 5, 4, 10, 10, 0, 0]])) assert_allclose(labels3[0], np.array([1, 0])) # Should raise a StopIteration with self.assertRaises(StopIteration): val_gen.__next__() def test_get_validation_data_default_word_and_character(self): get_val_gen, val_size = self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE], mode="word+character") val_gen = get_val_gen() assert val_size == 3 inputs1, labels1 = val_gen.__next__() assert_allclose(inputs1[0], np.array([2, 0])) assert_allclose( inputs1[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 10, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose(inputs1[2], np.array([3, 1])) assert_allclose( inputs1[3], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 11, 0, 0, 0], [6, 9, 2, 7, 8, 3, 5, 4, 1, 0, 0, 0]])) assert_allclose(labels1[0], np.array([1, 0])) inputs2, labels2 = val_gen.__next__() assert_allclose(inputs2[0], np.array([1, 0])) assert_allclose( inputs2[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose(inputs2[2], np.array([1, 0])) assert_allclose( inputs2[3], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 10, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose(labels2[0], np.array([0, 1])) inputs3, labels3 = val_gen.__next__() assert_allclose(inputs3[0], np.array([7, 0])) assert_allclose( inputs3[1], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 15, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])) assert_allclose(inputs3[2], np.array([8, 1])) assert_allclose( inputs3[3], np.array([[6, 9, 2, 7, 8, 3, 5, 4, 16, 0, 0, 0], [6, 9, 2, 7, 8, 3, 5, 4, 10, 10, 0, 0]])) assert_allclose(labels3[0], np.array([1, 0])) # Should raise a StopIteration with self.assertRaises(StopIteration): val_gen.__next__() def test_get_validation_data_pad_with_max_lens(self): get_val_gen, val_size = self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE], max_lengths={"num_sentence_words": 1}) val_gen = get_val_gen() assert val_size == 3 inputs1, labels1 = val_gen.__next__() assert_allclose(inputs1[0], np.array([2])) assert_allclose(inputs1[1], np.array([3])) assert_allclose(labels1[0], np.array([1, 0])) inputs2, labels2 = val_gen.__next__() assert_allclose(inputs2[0], np.array([1])) assert_allclose(inputs2[1], np.array([1])) assert_allclose(labels2[0], np.array([0, 1])) inputs3, labels3 = val_gen.__next__() assert_allclose(inputs3[0], np.array([7])) assert_allclose(inputs3[1], np.array([8])) assert_allclose(labels3[0], np.array([1, 0])) # Should raise a StopIteration with self.assertRaises(StopIteration): val_gen.__next__() def test_get_validation_data_with_max_features(self): get_val_gen, val_size = self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE], max_features=2) val_size == 2 val_gen = get_val_gen() inputs1, labels1 = val_gen.__next__() assert_allclose(inputs1[0], np.array([2, 0])) assert_allclose(inputs1[1], np.array([3, 1])) assert_allclose(labels1[0], np.array([1, 0])) inputs2, labels2 = val_gen.__next__() assert_allclose(inputs2[0], np.array([1, 0])) assert_allclose(inputs2[1], np.array([1, 0])) assert_allclose(labels2[0], np.array([0, 1])) # Should raise a StopIteration with self.assertRaises(StopIteration): val_gen.__next__() def test_get_validation_data_errors(self): with self.assertRaises(ValueError): self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE], max_lengths={"num_sentence_words": 1}, pad=False) with self.assertRaises(ValueError): self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE], max_lengths={"some wrong key": 1}) def test_get_validation_data_no_pad(self): get_val_gen, val_size = self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE], pad=False) assert val_size == 3 val_gen = get_val_gen() inputs1, labels1 = val_gen.__next__() assert_allclose(inputs1[0], np.array([2])) assert_allclose(inputs1[1], np.array([3, 1])) assert_allclose(labels1[0], np.array([1, 0])) inputs2, labels2 = val_gen.__next__() assert_allclose(inputs2[0], np.array([1])) assert_allclose(inputs2[1], np.array([1])) assert_allclose(labels2[0], np.array([0, 1])) inputs3, labels3 = val_gen.__next__() assert_allclose(inputs3[0], np.array([7])) assert_allclose(inputs3[1], np.array([8, 1, 1])) assert_allclose(labels3[0], np.array([1, 0])) # Should raise a StopIteration with self.assertRaises(StopIteration): val_gen.__next__() def test_generate_validation_batches(self): get_val_gen, val_size = self.data_manager.get_validation_data_from_file( [self.VALIDATION_FILE]) batch_gen = self.data_manager.get_batch_generator(get_val_gen, 2) new_batch_gen = DataManager.get_batch_generator(get_val_gen, 2) assert val_size == 3 # Assert that the new generator is a different object # than the old generator. assert new_batch_gen != batch_gen first_batch = batch_gen.__next__() new_first_batch = new_batch_gen.__next__() inputs, labels = first_batch new_inputs, new_labels = new_first_batch assert len(inputs) == len(new_inputs) == 2 assert len(labels) == len(new_labels) == 1 # Ensure output matches ground truth. assert_allclose(inputs[0], np.array([[2, 0], [1, 0]])) assert_allclose(inputs[1], np.array([[3, 1], [1, 0]])) assert_allclose(labels[0], np.array([[1, 0], [0, 1]])) # Ensure both generators produce same results. assert_allclose(inputs[0], new_inputs[0]) assert_allclose(inputs[1], new_inputs[1]) assert_allclose(labels[0], labels[0]) second_batch = batch_gen.__next__() new_second_batch = new_batch_gen.__next__() inputs, labels = second_batch new_inputs, new_labels = new_second_batch assert len(inputs) == len(new_inputs) == 2 assert len(labels) == len(new_labels) == 1 # Ensure output matches ground truth. assert_allclose(inputs[0], np.array([[7, 0]])) assert_allclose(inputs[1], np.array([[8, 1]])) assert_allclose(labels[0], np.array([[1, 0]])) # Ensure both generators produce same results. assert_allclose(inputs[0], new_inputs[0]) assert_allclose(inputs[1], new_inputs[1]) assert_allclose(labels[0], labels[0]) # Should raise a StopIteration with self.assertRaises(StopIteration): batch_gen.__next__() new_batch_gen.__next__()
# In[ ]: data_manager = DataManager(dataset=quora_dataset) # In[ ]: get_train_data_gen, train_data_size = data_manager.get_train_data_from_file() # In[ ]: get_val_data_gen, val_data_size = data_manager.get_validation_data_from_file() # In[ ]: # embedding_manager = EmbeddingManager(quora_dataset.data_indexer, pickle_dir='../models/') # embedding_matrix = embedding_manager.get_embedding_matrix(300,'../data/quora/external/glove.6B.300d.txt') nlp = spacy.load('en_core_web_md') embedding_matrix = EmbeddingManager.get_spacy_embedding_matrix(nlp) # In[ ]: from pympler import asizeof