def test_batch_size_two_fit_generator_test(self): """Tests the case when the batch size is two for fit_generator. The input text is specified in the above __init__ method. """ config = word_id_example_queue_config_pb2.WordIdExampleQueueConfig() config.file_name = self._input_text_file.name config.batch_size = 2 config.randomize_order = False config.fit_input_format = True word_id_example_queue = WordIdExampleQueue(config) expected_output_list = [] expected_output_list.append( # inputs part. ( np.array([[2, 10, 3, 8, 13, 9], [2, 11, 3, 8, 12, 9]]), # targets part. np.array([[10, 3, 8, 13, 9, 1], [11, 3, 8, 12, 9, 1]]))) expected_output_list.append( # inputs part. ( np.array([[2, 7, 5, 4, 3], [2, 7, 5, 4, 6]]), # targets part. np.array([[7, 5, 4, 3, 1], [7, 5, 4, 6, 1]]))) actual_output_list = [] for data in word_id_example_queue.dataset: actual_output_list.append(data) # The output contains two batches. 5 // 2 = 2. self.assertEqual(2, len(actual_output_list)) # Each batch has two elements, since the format is (inputs, targets) self.assertEqual(2, len(actual_output_list[0])) # Checks whether the batch size is two. self.assertEqual(2, actual_output_list[0][0].shape[0]) # Compares the contents of the actual outputs with the expected output. self.assertTrue( (expected_output_list[0][0] == actual_output_list[0][0].numpy() ).all()) self.assertTrue( (expected_output_list[0][1] == actual_output_list[0][1].numpy() ).all()) self.assertTrue( (expected_output_list[1][0] == actual_output_list[1][0].numpy() ).all()) self.assertTrue( (expected_output_list[1][1] == actual_output_list[1][1].numpy() ).all())
def test_batch_size_two_test(self): """Tests the case when the batch size is two. The input text is specified in the above __init__ method. """ config = word_id_example_queue_config_pb2.WordIdExampleQueueConfig() config.file_name = self._input_text_file.name config.batch_size = 2 config.randomize_order = False word_id_example_queue = WordIdExampleQueue(config) expected_output_list = [] expected_output_list.append( np.array([ [2, 10, 3, 8, 13, 9, 1], [2, 11, 3, 8, 12, 9, 1], ])) expected_output_list.append( np.array([ [2, 7, 5, 4, 3, 1], [2, 7, 5, 4, 6, 1], ])) actual_output_list = [] with tf.Session() as sess: sess.run(word_id_example_queue.initializer) while True: try: actual_output_list.append( sess.run(word_id_example_queue.get_batch())) except tf.errors.OutOfRangeError: break # The output contains two batches. 5 // 2 = 2. self.assertEqual(2, len(actual_output_list)) # Each batch is expected to contain two examples, because the batch # size is two. self.assertEqual(2, actual_output_list[0].shape[0]) # Compares the contents of the actual outputs with the expected output. self.assertTrue( (expected_output_list[0] == actual_output_list[0]).all()) self.assertTrue( (expected_output_list[1] == actual_output_list[1]).all())
def test_batch_size_one_test(self): """Tests the case when the batch size is one. The input text is specified in the above __init__ method. """ config = word_id_example_queue_config_pb2.WordIdExampleQueueConfig() config.file_name = self._input_text_file.name config.batch_size = 1 config.randomize_order = False word_id_example_queue = WordIdExampleQueue(config) expected_output_list = [] expected_output_list.append(np.array([2, 10, 3, 8, 13, 9, 1])) expected_output_list.append(np.array([2, 11, 3, 8, 12, 9, 1])) expected_output_list.append(np.array([2, 7, 5, 4, 3, 1])) expected_output_list.append(np.array([2, 7, 5, 4, 6, 1])) expected_output_list.append(np.array([2, 10, 6, 5, 4, 3, 1])) actual_output_list = [] with tf.Session() as sess: sess.run(word_id_example_queue.initializer) while True: try: actual_output_list.append( sess.run(word_id_example_queue.get_batch())) except tf.errors.OutOfRangeError: break # The output contains five batches. self.assertEqual(len(expected_output_list), len(actual_output_list)) # Each batch is expected to contain one example, because the batch # size is one. self.assertEqual(1, actual_output_list[0].shape[0]) index = 0 for expected_output in expected_output_list: self.assertTrue( (expected_output == actual_output_list[index]).all()) index += 1
def test_batch_size_two_test(self): """Tests the case when the batch size is two. The input text is specified in the above __init__ method. """ config = word_id_example_queue_config_pb2.WordIdExampleQueueConfig() config.file_name = self._input_text_file.name config.batch_size = 2 config.randomize_order = False config.fit_input_format = False word_id_example_queue = WordIdExampleQueue(config) expected_output_list = [] expected_output_list.append( np.array([ [2, 10, 3, 8, 13, 9, 1], [2, 11, 3, 8, 12, 9, 1], ])) expected_output_list.append( np.array([ [2, 7, 5, 4, 3, 1], [2, 7, 5, 4, 6, 1], ])) actual_output_list = [] for data in word_id_example_queue.dataset: actual_output_list.append(data.numpy()) # The output contains two batches. 5 // 2 = 2. self.assertEqual(2, len(actual_output_list)) # Each batch is expected to contain two examples, because the batch # size is two. self.assertEqual(2, actual_output_list[0].shape[0]) # Compares the contents of the actual outputs with the expected output. self.assertTrue( (expected_output_list[0] == actual_output_list[0]).all()) self.assertTrue( (expected_output_list[1] == actual_output_list[1]).all())
def test_batch_size_one_test(self): """Tests the case when the batch size is one. The input text is specified in the above __init__ method. """ config = word_id_example_queue_config_pb2.WordIdExampleQueueConfig() config.file_name = self._input_text_file.name config.batch_size = 1 config.randomize_order = False config.fit_input_format = False word_id_example_queue = WordIdExampleQueue(config) expected_output_list = [] expected_output_list.append(np.array([2, 10, 3, 8, 13, 9, 1])) expected_output_list.append(np.array([2, 11, 3, 8, 12, 9, 1])) expected_output_list.append(np.array([2, 7, 5, 4, 3, 1])) expected_output_list.append(np.array([2, 7, 5, 4, 6, 1])) expected_output_list.append(np.array([2, 10, 6, 5, 4, 3, 1])) actual_output_list = [] for data in word_id_example_queue.dataset: actual_output_list.append(data.numpy()) # The output contains five batches. self.assertEqual(len(expected_output_list), len(actual_output_list)) # Each batch is expected to contain one example, because the batch # size is one. self.assertEqual(1, actual_output_list[0].shape[0]) index = 0 for expected_output in expected_output_list: self.assertTrue( (expected_output == actual_output_list[index]).all()) index += 1
#self._dropout_2 = tf.keras.layers.Dropout( # dropout_rate, noise_shape=(input_shape[0], 1, cell_size)), self._softmax = tf.keras.layers.Dense( target_size, activation="softmax", ) def call(self, inputs, training=False): output = self._embedding(inputs) output = self._lstm_0(output) output = self._softmax(output) return (output) config = word_id_example_queue_config_pb2.WordIdExampleQueueConfig() # TODO TODO (The following needs to be fixed) config.file_name = "/home/chanwcom/chanwcom_local_repository/star_organization/speech01/speech/speech/trainer/keras_lm_trainer/data/ptb.train.txt" config.batch_size = 20 config.randomize_order = False word_id_example_queue = WordIdExampleQueue(config) #TODO TODO(chanw.com) Adds a routine to construct the example queue. #target_size = 10002 target_size = 10010 cell_size = 256 dropout_rate = 0.5