def testParsingReaderOp(self): # Runs the reader over the test input for two epochs. num_steps_a = 0 num_actions = 0 num_word_ids = 0 num_tag_ids = 0 num_label_ids = 0 batch_size = 10 with self.test_session() as sess: (words, tags, labels), epochs, gold_actions = (gen_parser_ops.gold_parse_reader( self._task_context, 3, batch_size, corpus_name='training-corpus')) while True: tf_gold_actions, tf_epochs, tf_words, tf_tags, tf_labels = ( sess.run([gold_actions, epochs, words, tags, labels])) num_steps_a += 1 num_actions = max(num_actions, max(tf_gold_actions) + 1) num_word_ids = max(num_word_ids, self.GetMaxId(tf_words) + 1) num_tag_ids = max(num_tag_ids, self.GetMaxId(tf_tags) + 1) num_label_ids = max(num_label_ids, self.GetMaxId(tf_labels) + 1) self.assertIn(tf_epochs, [0, 1, 2]) if tf_epochs > 1: break # Runs the reader again, this time with a lot of added graph nodes. num_steps_b = 0 with self.test_session() as sess: num_features = [6, 6, 4] num_feature_ids = [num_word_ids, num_tag_ids, num_label_ids] embedding_sizes = [8, 8, 8] hidden_layer_sizes = [32, 32] # Here we aim to test the iteration of the reader op in a complex network, # not the GraphBuilder. parser = graph_builder.GreedyParser(num_actions, num_features, num_feature_ids, embedding_sizes, hidden_layer_sizes) parser.AddTraining(self._task_context, batch_size, corpus_name='training-corpus') sess.run(list(parser.inits.values())) while True: tf_epochs, tf_cost, _ = sess.run([ parser.training['epochs'], parser.training['cost'], parser.training['train_op'] ]) num_steps_b += 1 self.assertGreaterEqual(tf_cost, 0) self.assertIn(tf_epochs, [0, 1, 2]) if tf_epochs > 1: break # Assert that the two runs made the exact same number of steps. logging.info('Number of steps in the two runs: %d, %d', num_steps_a, num_steps_b) self.assertEqual(num_steps_a, num_steps_b)
def testParsingReaderOp(self): # Runs the reader over the test input for two epochs. num_steps_a = 0 num_actions = 0 num_word_ids = 0 num_tag_ids = 0 num_label_ids = 0 batch_size = 10 with self.test_session() as sess: (words, tags, labels), epochs, gold_actions = ( gen_parser_ops.gold_parse_reader(self._task_context, 3, batch_size, corpus_name='training-corpus')) while True: tf_gold_actions, tf_epochs, tf_words, tf_tags, tf_labels = ( sess.run([gold_actions, epochs, words, tags, labels])) num_steps_a += 1 num_actions = max(num_actions, max(tf_gold_actions) + 1) num_word_ids = max(num_word_ids, self.GetMaxId(tf_words) + 1) num_tag_ids = max(num_tag_ids, self.GetMaxId(tf_tags) + 1) num_label_ids = max(num_label_ids, self.GetMaxId(tf_labels) + 1) self.assertIn(tf_epochs, [0, 1, 2]) if tf_epochs > 1: break # Runs the reader again, this time with a lot of added graph nodes. num_steps_b = 0 with self.test_session() as sess: num_features = [6, 6, 4] num_feature_ids = [num_word_ids, num_tag_ids, num_label_ids] embedding_sizes = [8, 8, 8] hidden_layer_sizes = [32, 32] # Here we aim to test the iteration of the reader op in a complex network, # not the GraphBuilder. parser = graph_builder.GreedyParser( num_actions, num_features, num_feature_ids, embedding_sizes, hidden_layer_sizes) parser.AddTraining(self._task_context, batch_size, corpus_name='training-corpus') sess.run(parser.inits.values()) while True: tf_epochs, tf_cost, _ = sess.run( [parser.training['epochs'], parser.training['cost'], parser.training['train_op']]) num_steps_b += 1 self.assertGreaterEqual(tf_cost, 0) self.assertIn(tf_epochs, [0, 1, 2]) if tf_epochs > 1: break # Assert that the two runs made the exact same number of steps. logging.info('Number of steps in the two runs: %d, %d', num_steps_a, num_steps_b) self.assertEqual(num_steps_a, num_steps_b)
def _AddGoldReader(self, task_context, batch_size, corpus_name): features, epochs, gold_actions = ( gen_parser_ops.gold_parse_reader(task_context, self._feature_size, batch_size, corpus_name=corpus_name, arg_prefix=self._arg_prefix)) return {'gold_actions': tf.identity(gold_actions, name='gold_actions'), 'epochs': tf.identity(epochs, name='epochs'), 'feature_endpoints': features}
def ParserEndpoints(): return gen_parser_ops.gold_parse_reader(self._task_context, feature_size, batch_size, corpus_name='training-corpus')
def ParserEndpoints(): return gen_parser_ops.gold_parse_reader( self._task_context, feature_size, batch_size, corpus_name='training-corpus')