def export(): checkpoint_path = FLAGS.checkpoint_path export_path = FLAGS.export_path vocab_path = FLAGS.vocab_path num_steps = FLAGS.num_steps vocab_size = FLAGS.vocab_size embedding_size = FLAGS.embedding_size hidden_size = FLAGS.hidden_size keep_prob = FLAGS.keep_prob num_layers = FLAGS.num_layers num_classes = FLAGS.num_classes prop_limit = FLAGS.prop_limit # split 1-D String dense Tensor to words SparseTensor sentences = tf.placeholder(dtype=tf.string, shape=[None], name='input_sentences') sparse_words = tf.string_split(sentences, delimiter=' ') # slice SparseTensor valid_indices = tf.less(sparse_words.indices, tf.constant([num_steps], dtype=tf.int64)) valid_indices = tf.reshape( tf.split(valid_indices, [1, 1], axis=1)[1], [-1]) valid_sparse_words = tf.sparse_retain(sparse_words, valid_indices) excess_indices = tf.greater_equal(sparse_words.indices, tf.constant([num_steps], dtype=tf.int64)) excess_indices = tf.reshape( tf.split(excess_indices, [1, 1], axis=1)[1], [-1]) excess_sparse_words = tf.sparse_retain(sparse_words, excess_indices) # sparse to dense words = tf.sparse_to_dense( sparse_indices=valid_sparse_words.indices, output_shape=[valid_sparse_words.dense_shape[0], num_steps], sparse_values=valid_sparse_words.values, default_value='_PAD') # dict words to token ids words_table = lookup.index_table_from_file(os.path.join( vocab_path, 'words_vocab.txt'), default_value=3) words_ids = words_table.lookup(words) # blstm model predict with tf.variable_scope('model', reuse=None): logits, _ = ner_model.inference(words_ids, valid_sparse_words.dense_shape[0], num_steps, vocab_size, embedding_size, hidden_size, keep_prob, num_layers, num_classes, is_training=False) props = tf.nn.softmax(logits) max_prop_values, max_prop_indices = tf.nn.top_k(props, k=1) predict_scores = tf.reshape(max_prop_values, shape=[-1, num_steps]) predict_labels_ids = tf.reshape(max_prop_indices, shape=[-1, num_steps]) predict_labels_ids = tf.to_int64(predict_labels_ids) # replace untrusted prop that less than prop_limit trusted_prop_flag = tf.greater_equal( predict_scores, tf.constant(prop_limit, dtype=tf.float32)) replace_prop_labels_ids = tf.to_int64( tf.fill(tf.shape(predict_labels_ids), 4)) predict_labels_ids = tf.where(trusted_prop_flag, predict_labels_ids, replace_prop_labels_ids) # dict token ids to labels labels_table = lookup.index_to_string_table_from_file(os.path.join( vocab_path, 'labels_vocab.txt'), default_value='o') predict_labels = labels_table.lookup(predict_labels_ids) # extract real blstm predict label in dense and save to sparse valid_sparse_predict_labels = tf.SparseTensor( indices=valid_sparse_words.indices, values=tf.gather_nd(predict_labels, valid_sparse_words.indices), dense_shape=valid_sparse_words.dense_shape) # create excess label SparseTensor with 'O' excess_sparse_predict_labels = tf.SparseTensor( indices=excess_sparse_words.indices, values=tf.fill(tf.shape(excess_sparse_words.values), 'O'), dense_shape=excess_sparse_words.dense_shape) # concat SparseTensor sparse_predict_labels = tf.SparseTensor( indices=tf.concat(axis=0, values=[ valid_sparse_predict_labels.indices, excess_sparse_predict_labels.indices ]), values=tf.concat(axis=0, values=[ valid_sparse_predict_labels.values, excess_sparse_predict_labels.values ]), dense_shape=excess_sparse_predict_labels.dense_shape) sparse_predict_labels = tf.sparse_reorder(sparse_predict_labels) # join SparseTensor to 1-D String dense Tensor # remain issue, num_split should equal the real size, but here limit to 1 join_labels_list = [] slice_labels_list = tf.sparse_split(sp_input=sparse_predict_labels, num_split=1, axis=0) for slice_labels in slice_labels_list: slice_labels = slice_labels.values join_labels = tf.reduce_join(slice_labels, reduction_indices=0, separator=' ') join_labels_list.append(join_labels) format_predict_labels = tf.stack(join_labels_list) saver = tf.train.Saver() with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(checkpoint_path) if ckpt and ckpt.model_checkpoint_path: print('read model from {}'.format(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) global_step = int(ckpt.model_checkpoint_path.split('-')[-1]) else: print('No checkpoint file found at %s' % FLAGS.checkpoint_path) return # Export inference model. output_path = os.path.join(export_path, str(global_step)) print 'Exporting trained model to', output_path builder = tf.saved_model.builder.SavedModelBuilder(output_path) # Build the signature_def_map. predict_inputs_tensor_info = tf.saved_model.utils.build_tensor_info( sentences) predict_output_tensor_info = tf.saved_model.utils.build_tensor_info( format_predict_labels) prediction_signature = tf.saved_model.signature_def_utils.build_signature_def( inputs={ 'input_sentences': predict_inputs_tensor_info, }, outputs={'classes': predict_output_tensor_info}, method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME) legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') builder.add_meta_graph_and_variables( sess, [tf.saved_model.tag_constants.SERVING], signature_def_map={'predict_ner': prediction_signature}, legacy_init_op=legacy_init_op) builder.save() print 'Successfully exported model to %s' % export_path
def run_lstm(embedding): example_batch = input_pipeline(['data/train_examples.tfrecords'], batch_size=64) lengths, questions, answers, images = example_batch final_state = lstm(questions, lengths, embedding) with tf.variable_scope('mlp'): weights = tf.get_variable('weights', [512, 28], initializer=tf.random_normal_initializer()) biases = tf.get_variable('biases', [28], initializer=tf.zeros_initializer()) out = tf.nn.xw_plus_b(final_state, weights, biases) with tf.variable_scope('eval'): predictions = tf.argmax(tf.nn.softmax(out), axis=1) correct = tf.equal(predictions, answers) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) tf.summary.scalar('accuracy', accuracy) loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits=out, labels=answers)) with tf.variable_scope('train'): train_op = tf.train.AdamOptimizer(1e-4).minimize(loss) with tf.name_scope('Text_Decode'): question_table = index_to_string_table_from_file( vocabulary_file='data/vocabulary.txt', name='Question_Table') answer_table = index_to_string_table_from_file( vocabulary_file='data/answers.txt', name='Answer_Table') question_strings = tf.expand_dims(tf.reduce_join(question_table.lookup( tf.slice(questions, [0, 0], [5, -1])), axis=1, separator=' '), axis=1) answer_strings = tf.expand_dims(answer_table.lookup( tf.slice(answers, [0], [5])), axis=1) prediction_strings = tf.expand_dims(answer_table.lookup( tf.slice(predictions, [0], [5])), axis=1) labels = tf.constant(['Question', 'Answer', 'Prediction'], shape=[1, 3]) qa_table = tf.concat( [question_strings, answer_strings, prediction_strings], axis=1) qa_table = tf.concat([labels, qa_table], axis=0) print(qa_table) # qa_string = tf.string_join([qa_string, prediction_strings], separator='\r\nPredicted: ') tf.summary.text('Question', qa_table) for var in tf.trainable_variables(): tf.summary.histogram(var.op.name, var) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() with tf.Session() as sess: tf.tables_initializer().run() sess.run(init_op) summary_writer = tf.summary.FileWriter('logs/3', graph=sess.graph) summary_op = tf.summary.merge_all() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) for i in range(5000001): if i % 100 == 0: _, summary, acc = sess.run([train_op, summary_op, accuracy]) summary_writer.add_summary(summary, i) print('\rStep %d, Accuracy: %f' % (i, acc), end='' * 10) else: sess.run(train_op) print('\rStep %d' % i, end=' ' * 20) if i % 5000 == 0: saver.save(sess, 'logs/3/model.ckpt', global_step=i) coord.request_stop() coord.join(threads)
def main(embedding): # img_files = tf.gfile.ListDirectory(img_dir) # print(img_files) # img_files = [os.path.join(img_dir, filename) for filename in img_files] # filename_queue = tf.train.string_input_producer(img_files, shuffle=False, num_epochs=1) # # img_reader = tf.WholeFileReader() # # name, img_data = img_reader.read(filename_queue) # name = filename_queue.dequeue() id_to_label = {} with open(os.path.join('id_to_human.json')) as file: id_to_label = json.load(file) extract_pretrained_weights('data/resnet_v1_101.ckpt') example_batch = input_pipeline(['data/train_examples.tfrecords'], batch_size=32) lengths, questions, answers, images = example_batch logits = res_net(images) # image_cat = tf.argmax(tf.nn.softmax(logits), axis=1) last_hidden = lstm(questions, lengths, embedding) print(logits) print(last_hidden) output = mlp_layer(logits, last_hidden) with tf.name_scope('inference'): predictions = tf.argmax(tf.nn.softmax(output), axis=1) correct = tf.equal(predictions, answers) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) tf.summary.scalar('accuracy', accuracy) # print(predictions) # print(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)) # print(tf.get_collection(tf.GraphKeys.UPDATE_OPS)) train_op = train_ops(output, answers) with tf.name_scope('Text_Decode'): question_table = index_to_string_table_from_file( vocabulary_file='data/vocabulary.txt', name='Question_Table') answer_table = index_to_string_table_from_file( vocabulary_file='data/answers.txt', name='Answer_Table') question_strings = tf.expand_dims(tf.reduce_join(question_table.lookup( tf.slice(questions, [0, 0], [5, -1])), axis=1, separator=' '), axis=1) answer_strings = tf.expand_dims(answer_table.lookup( tf.slice(answers, [0], [5])), axis=1) prediction_strings = tf.expand_dims(answer_table.lookup( tf.slice(predictions, [0], [5])), axis=1) labels = tf.constant(['Question', 'Answer', 'Prediction'], shape=[1, 3]) qa_table = tf.concat( [question_strings, answer_strings, prediction_strings], axis=1) qa_table = tf.concat([labels, qa_table], axis=0) print(qa_table) # qa_string = tf.string_join([qa_string, prediction_strings], separator='\r\nPredicted: ') tf.summary.text('Question', qa_table) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) saver = tf.train.Saver() with tf.Session() as sess: tf.tables_initializer().run() sess.run(init_op) summary_writer = tf.summary.FileWriter('logs/6', graph=sess.graph) summary_op = tf.summary.merge_all() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) for i in range(5000001): if i % 100 == 0: _, summary, acc = sess.run([train_op, summary_op, accuracy]) summary_writer.add_summary(summary, i) print('\rStep %d, Accuracy: %f' % (i, acc)) else: sess.run(train_op) print('\rStep %d' % i, end=' ' * 20) if i % 5000 == 0: saver.save(sess, 'logs/6/model.ckpt', global_step=i) coord.request_stop() coord.join(threads)
def model_fn(self, features, labels, params, mode): # embedding input and target sequence vocab_size = params["vocab_size"] num_units = params["num_units"] input_keep_prob = params["input_keep_prob"] layer_size = params["encode_layzer_size"] in_seq = features["input"] in_seq_length = features["input_length"] if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL: out_seq = labels["output_out"] out_seq_length = labels["output_length"] else: out_seq = None out_seq_length = None table = lookup.index_table_from_file(params["vocabs_labels_file"], default_value=0) features_ids = table.lookup(in_seq) with tf.variable_scope("embedding"): embedding = tf.Variable(tf.random.uniform([vocab_size, num_units], -1.0, 1.0), dtype=tf.float32, name='word_embedding') # embedding = tf.get_variable(name='embed', shape=[vocab_size, num_units]) embed_input = tf.nn.embedding_lookup(params=embedding, ids=features_ids, name="embed_input") # encode and decode bi_layer_size = int(layer_size / 2) encoder_cell_fw = self.get_layered_cell(bi_layer_size, num_units, input_keep_prob) encoder_cell_bw = self.get_layered_cell(bi_layer_size, num_units, input_keep_prob) # sequence_length 应该是序列真实的长度 bi_encoder_output, bi_encoder_state = tf.nn.bidirectional_dynamic_rnn( cell_fw=encoder_cell_fw, cell_bw=encoder_cell_bw, inputs=embed_input, sequence_length=in_seq_length, dtype=embed_input.dtype, time_major=False) # concat encode output and state encoder_output = tf.concat(bi_encoder_output, -1) encoder_state = [] for layers_id in range(bi_layer_size): encoder_state.append(bi_encoder_state[0][layers_id]) encoder_state.append(bi_encoder_state[1][layers_id]) encoder_state = tuple(encoder_state) decoder_cell = self.attention_decoder_cell(encoder_output, in_seq_length, num_units, layer_size, input_keep_prob) batch_size = tf.shape(in_seq_length)[0] init_state = decoder_cell.zero_state( batch_size, tf.float32).clone(cell_state=encoder_state) # str2int labels_ids = None if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL: labels_ids = table.lookup(out_seq) embed_target = tf.nn.embedding_lookup(embedding, labels_ids, name="embed_target") helper = tf.contrib.seq2seq.TrainingHelper(embed_target, out_seq_length, time_major=False) else: # TODO: start tokens and end tokens are hard code helper = tf.contrib.seq2seq.GreedyEmbeddingHelper( embedding, tf.fill([batch_size], 0), 1) predict_output_layer = core.Dense(vocab_size, use_bias=False) basic_decoder = tf.contrib.seq2seq.BasicDecoder( cell=decoder_cell, helper=helper, initial_state=init_state, output_layer=predict_output_layer) # 执行动态解码 (final_outputs, final_state, final_sequence_lengths) final_outputs, final_state, final_sequence_lengths = tf.contrib.seq2seq.dynamic_decode( decoder=basic_decoder, maximum_iterations=50) loss = None train_op = None predictions = None if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL: outputs = final_outputs.rnn_output loss = self.compute_loss(outputs, labels_ids, out_seq_length) trainable_params = tf.trainable_variables() global_step = tf.train.get_global_step() clipped_gradients, _ = tf.clip_by_global_norm( tf.gradients(loss, trainable_params), 0.8) train_op = tf.train.AdamOptimizer( learning_rate=0.001).apply_gradients( zip(clipped_gradients, trainable_params), global_step=global_step) # train_op = tf.train.AdamOptimizer(learning_rate=0.01).minimize(loss) # tf.summary.scalar('loss', loss) tf.add_to_collection("loss", loss) else: outputs = final_outputs.sample_id predict_ids = tf.cast(outputs, dtype=tf.int64) mask = tf.sequence_mask(in_seq_length, maxlen=tf.shape(predict_ids)[1]) real_ids = predict_ids * tf.cast(mask, dtype=tf.int64) index_to_string = lookup.index_to_string_table_from_file( params["vocabs_labels_file"]) predict_labels = index_to_string.lookup(real_ids) predictions = { "predict_labels": predict_labels, "predict_ids": real_ids, } tf.add_to_collection("predictions", predictions) return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, loss=loss, train_op=train_op)
def testModel(self): # 数据输入 dataset_f = tf.data.TextLineDataset(filenames=FEATURE_FILE) dataset_l = tf.data.TextLineDataset(filenames=LABEL_FILE) dataset = tf.data.Dataset.zip((dataset_f, dataset_l)) dataset = dataset.map(lambda x, y: (tf.string_split([x], delimiter=",").values, tf.string_split([y], delimiter=",").values)) dataset = dataset.shuffle(1000) dataset = dataset.map(lambda x, y: (x, tf.size(x), y)) dataset = dataset.padded_batch( batch_size=2, padded_shapes=([ None, ], [], [None]), padding_values=(tf.constant("<UNK>", dtype=tf.string), 0, tf.constant("O", dtype=tf.string))) iterator = dataset.make_initializable_iterator() tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer) feature, feature_length, label = iterator.get_next() # 通过字典获得索引 words2idx = lookup_op.index_table_from_file(VOCAB_FEATURE) words_ids = words2idx.lookup(feature) # 编码 with tf.variable_scope("embedding_test", reuse=tf.AUTO_REUSE): embedding_variable = tf.get_variable("embedding_variable", shape=[18, 256], dtype=tf.double) embedding = tf.nn.embedding_lookup(embedding_variable, words_ids) embedding_outputs = tf.layers.dropout(inputs=embedding, rate=0.5, training=True) inputs = embedding_outputs # 模型 with tf.variable_scope("lstm_model", reuse=tf.AUTO_REUSE): cells = [ tf.nn.rnn_cell.LSTMCell(256), tf.nn.rnn_cell.LSTMCell(256) ] cell = tf.nn.rnn_cell.MultiRNNCell(cells=cells) outputs, state = tf.nn.dynamic_rnn(cell=cell, inputs=inputs, sequence_length=feature_length, dtype=inputs.dtype) # 输出层 logits = tf.layers.dense(inputs=outputs, units=5) # 解码 transition = tf.get_variable(name="transition", shape=[5, 5], dtype=tf.double) decode_ids, best_score = tf.contrib.crf.crf_decode( logits, transition, feature_length) # 通过索引获得标签 idx2tags = lookup_op.index_to_string_table_from_file(VOCAB_LABEL, default_value="O") tags = idx2tags.lookup(tf.cast(decode_ids, dtype=tf.int64)) # 计算损失 tags2idx = lookup_op.index_table_from_file(VOCAB_LABEL) actual_ids = tags2idx.lookup(label) log_likeilhood, _ = tf.contrib.crf.crf_log_likelihood( inputs=tf.cast(logits, dtype=tf.float32), tag_indices=actual_ids, sequence_lengths=feature_length) loss = tf.reduce_mean(-log_likeilhood) with self.test_session() as sess: sess.run(tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS)) sess.run(tf.global_variables_initializer()) # print(sess.run(feature)) # print(sess.run(feature_length)) # print(sess.run(label)) # print(sess.run(words2idx)) # print(sess.run(words_ids)) # print(sess.run(embedding_variable)) # print(sess.run(tf.shape(embedding_variable))) # print(sess.run(embedding)) # print(sess.run(tf.shape(embedding))) # print(sess.run(embedding_outputs)) # print(sess.run(tf.shape(embedding_outputs))) # print(sess.run(outputs)) # print(sess.run(tf.shape(outputs))) # print(sess.run(tf.shape(state[0]))) # print(sess.run(logits)) # print(sess.run(tf.shape(logits))) # print(sess.run(decode_tags)) # print(sess.run(tags)) # print(sess.run(actual_ids)) # print(sess.run(log_likeilhood)) print(sess.run(loss)) return
def tensor_predict(words_list): num_classes = FLAGS.num_classes num_layers = FLAGS.num_layers num_steps = FLAGS.num_steps embedding_size = FLAGS.embedding_size hidden_size = FLAGS.hidden_size keep_prob = FLAGS.keep_prob vocab_size = FLAGS.vocab_size vocab_path = FLAGS.vocab_path prop_limit = FLAGS.prop_limit checkpoint_path = FLAGS.checkpoint_path # split 1-D String dense Tensor to words SparseTensor sentences = tf.placeholder(dtype=tf.string, shape=[None], name='input_sentences') sparse_words = tf.string_split(sentences, delimiter=' ') # slice SparseTensor valid_indices = tf.less(sparse_words.indices, tf.constant([num_steps], dtype=tf.int64)) valid_indices = tf.reshape( tf.split(valid_indices, [1, 1], axis=1)[1], [-1]) valid_sparse_words = tf.sparse_retain(sparse_words, valid_indices) excess_indices = tf.greater_equal(sparse_words.indices, tf.constant([num_steps], dtype=tf.int64)) excess_indices = tf.reshape( tf.split(excess_indices, [1, 1], axis=1)[1], [-1]) excess_sparse_words = tf.sparse_retain(sparse_words, excess_indices) # sparse to dense words = tf.sparse_to_dense( sparse_indices=valid_sparse_words.indices, output_shape=[valid_sparse_words.dense_shape[0], num_steps], sparse_values=valid_sparse_words.values, default_value='_PAD') # dict words to token ids # with open(os.path.join(vocab_path, 'words_vocab.txt'), 'r') as data_file: # words_table_list = [line.strip() for line in data_file if line.strip()] # words_table_tensor = tf.constant(words_table_list, dtype=tf.string) # words_table = lookup.index_table_from_tensor(mapping=words_table_tensor, default_value=3) words_table = lookup.index_table_from_file(os.path.join( vocab_path, 'words_vocab.txt'), default_value=3) words_ids = words_table.lookup(words) # blstm model predict with tf.variable_scope('model', reuse=None): logits, _ = model.inference(words_ids, valid_sparse_words.dense_shape[0], num_steps, vocab_size, embedding_size, hidden_size, keep_prob, num_layers, num_classes, is_training=False) # using softmax # props = tf.nn.softmax(logits) # max_prop_values, max_prop_indices = tf.nn.top_k(props, k=1) # predict_scores = tf.reshape(max_prop_values, shape=[-1, num_steps]) # predict_labels_ids = tf.reshape(max_prop_indices, shape=[-1, num_steps]) # predict_labels_ids = tf.to_int64(predict_labels_ids) # using crf logits = tf.reshape(logits, shape=[-1, num_steps, num_classes]) transition_params = tf.get_variable("transitions", [num_classes, num_classes]) sequence_length = tf.constant(num_steps, shape=[logits.get_shape()[0]], dtype=tf.int64) predict_labels_ids, _ = crf_utils.crf_decode(logits, transition_params, sequence_length) predict_labels_ids = tf.to_int64(predict_labels_ids) predict_scores = tf.constant(1.0, shape=predict_labels_ids.get_shape(), dtype=tf.float32) # replace untrusted prop that less than prop_limit trusted_prop_flag = tf.greater_equal( predict_scores, tf.constant(prop_limit, dtype=tf.float32)) replace_prop_labels_ids = tf.to_int64( tf.fill(tf.shape(predict_labels_ids), 4)) predict_labels_ids = tf.where(trusted_prop_flag, predict_labels_ids, replace_prop_labels_ids) # dict token ids to labels # with open(os.path.join(vocab_path, 'labels_vocab.txt'), 'r') as data_file: # labels_table_list = [line.strip() for line in data_file if line.strip()] # labels_table_tensor = tf.constant(labels_table_list, dtype=tf.string) # labels_table = lookup.index_to_string_table_from_tensor(mapping=labels_table_tensor, default_value='O') labels_table = lookup.index_to_string_table_from_file(os.path.join( vocab_path, 'labels_vocab.txt'), default_value='O') predict_labels = labels_table.lookup(predict_labels_ids) # extract real blstm predict label in dense and save to sparse valid_sparse_predict_labels = tf.SparseTensor( indices=valid_sparse_words.indices, values=tf.gather_nd(predict_labels, valid_sparse_words.indices), dense_shape=valid_sparse_words.dense_shape) # create excess label SparseTensor with 'O' excess_sparse_predict_labels = tf.SparseTensor( indices=excess_sparse_words.indices, values=tf.fill(tf.shape(excess_sparse_words.values), 'O'), dense_shape=excess_sparse_words.dense_shape) # concat SparseTensor sparse_predict_labels = tf.SparseTensor( indices=tf.concat(axis=0, values=[ valid_sparse_predict_labels.indices, excess_sparse_predict_labels.indices ]), values=tf.concat(axis=0, values=[ valid_sparse_predict_labels.values, excess_sparse_predict_labels.values ]), dense_shape=excess_sparse_predict_labels.dense_shape) sparse_predict_labels = tf.sparse_reorder(sparse_predict_labels) # join SparseTensor to 1-D String dense Tensor # remain issue, num_split should equal the real size, but here limit to 1 join_labels_list = [] slice_labels_list = tf.sparse_split(sp_input=sparse_predict_labels, num_split=1, axis=0) for slice_labels in slice_labels_list: slice_labels = slice_labels.values join_labels = tf.reduce_join(slice_labels, reduction_indices=0, separator=' ') join_labels_list.append(join_labels) format_predict_labels = tf.stack(join_labels_list, name='predict_labels') saver = tf.train.Saver() tables_init_op = tf.tables_initializer() with tf.Session() as sess: sess.run(tables_init_op) ckpt = tf.train.get_checkpoint_state(checkpoint_path) if ckpt and ckpt.model_checkpoint_path: print('read model from {}'.format(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found at %s' % checkpoint_path) return # crf tensor predict_labels_list = sess.run(format_predict_labels, feed_dict={sentences: words_list}) # save graph into .pb file graph = tf.graph_util.convert_variables_to_constants( sess, sess.graph_def, ["init_all_tables", "predict_labels"]) tf.train.write_graph(graph, '.', 'ner_graph.pb', as_text=False) return predict_labels_list
def vocabulary_lookup_reverse(self): # build a table for ids2word return lookup.index_to_string_table_from_file( vocabulary_file=self.vocabulary_file, vocab_size=self.size - self.num_oov_buckets, default_value=UNKNOWN_TOKEN)
def _build_graph_with_datasets(self, train_minibatch, dev_minibatch, test_minibatch, num_classes): """Returns the training/evaluation ops.""" self.keep_prob = 1. # Using literal 1 (not placeholder) skips dropout op. self.sentence_representer._keep_prob = 1. # pylint:disable=protected-access # Build the graph as per the base class. (train_joint_rep, _) = self.add_representation(train_minibatch) (train_accuracy, loss, opt_step) = self.add_train_ops(num_classes, train_joint_rep, train_minibatch) dev_num_correct = self.add_dev_eval_ops(dev_minibatch) test_num_correct = self.add_dev_eval_ops(test_minibatch) graph_tensors = { 'loss': loss, 'train_op': opt_step, 'train_accuracy': train_accuracy, 'dev_num_correct': dev_num_correct, 'test_num_correct': test_num_correct, 'keep_prob': self.keep_prob } vocab_table = self.embed_pad.vocab_table vocab_size = self.embed_pad.vocab_size verifiable_loss_ratio = tf.constant( self.config['verifiable_loss_ratio'], dtype=tf.float32, name='verifiable_loss_ratio') self.delta = tf.constant(self.config['delta'], dtype=tf.float32, name='delta') lookup_token = tf.placeholder(tf.string, shape=None, name='lookup_token') indices = vocab_table.lookup(lookup_token) self.vocab_list = contrib_lookup.index_to_string_table_from_file( self.config['vocab_filename_pad']) lookup_token_index = tf.placeholder(tf.int64, shape=None, name='lookup_token_index') lookup_token_string = self.vocab_list.lookup(lookup_token_index) synonym_values = tf.placeholder(tf.int64, shape=[None, None], name='synonym_values') synonym_counts = tf.placeholder(tf.int64, shape=[None], name='synonym_counts') train_perturbation = self.create_perturbation_ops( train_minibatch, synonym_values, vocab_table) train_data_batch, _, _ = self.preprocess_databatch( train_minibatch, vocab_table, train_perturbation) train_words = self.vocab_list.lookup(train_data_batch.input_tokens) # [num_targets x batchsize] verifiable_obj = self.add_verifiable_objective(train_minibatch, vocab_table, train_perturbation, stop_gradient=False) train_nominal = self.run_classification( train_data_batch.embedded_inputs, train_data_batch.sentiment, train_data_batch.length) train_bound, train_verified = self.compute_verifiable_verified( verifiable_obj) verifiable_loss = self.compute_verifiable_loss( verifiable_obj, train_minibatch.sentiment) if (self.config['verifiable_loss_ratio']) > 1.0: raise ValueError('Loss ratios sum up to more than 1.0') total_loss = (1 - verifiable_loss_ratio) * graph_tensors['loss'] if self.config['verifiable_loss_ratio'] != 0: total_loss += verifiable_loss_ratio * verifiable_loss # Attack on dev/test set. dev_perturbation = self.create_perturbation_ops( dev_minibatch, synonym_values, vocab_table) # [num_targets x batchsize] dev_verifiable_obj = self.add_verifiable_objective(dev_minibatch, vocab_table, dev_perturbation, stop_gradient=True) dev_bound, dev_verified = self.compute_verifiable_verified( dev_verifiable_obj) dev_data_batch, _, _ = self.preprocess_databatch( dev_minibatch, vocab_table, dev_perturbation) test_perturbation = self.create_perturbation_ops( test_minibatch, synonym_values, vocab_table) # [num_targets x batchsize] test_verifiable_obj = self.add_verifiable_objective(test_minibatch, vocab_table, test_perturbation, stop_gradient=True) test_bound, test_verified = self.compute_verifiable_verified( test_verifiable_obj) test_data_batch, _, _ = self.preprocess_databatch( test_minibatch, vocab_table, test_perturbation) dev_words = self.vocab_list.lookup(dev_data_batch.input_tokens) test_words = self.vocab_list.lookup(test_data_batch.input_tokens) dev_nominal = self.run_classification(dev_data_batch.embedded_inputs, dev_data_batch.sentiment, dev_data_batch.length) test_nominal = self.run_classification(test_data_batch.embedded_inputs, test_data_batch.sentiment, test_data_batch.length) dev_predictions = self.run_prediction(dev_data_batch.embedded_inputs, dev_data_batch.length) test_predictions = self.run_prediction(test_data_batch.embedded_inputs, test_data_batch.length) with tf.control_dependencies( [train_verified, test_verified, dev_verified]): opt_step = self._add_optimize_op(total_loss) graph_tensors['total_loss'] = total_loss graph_tensors['verifiable_loss'] = verifiable_loss graph_tensors['train_op'] = opt_step graph_tensors['indices'] = indices graph_tensors['lookup_token_index'] = lookup_token_index graph_tensors['lookup_token_string'] = lookup_token_string graph_tensors['lookup_token'] = lookup_token graph_tensors['vocab_size'] = vocab_size graph_tensors['synonym_values'] = synonym_values graph_tensors['synonym_counts'] = synonym_counts graph_tensors['verifiable_loss_ratio'] = verifiable_loss_ratio graph_tensors['delta'] = self.delta graph_tensors['train'] = { 'bound': train_bound, 'verified': train_verified, 'words': train_words, 'sentiment': train_minibatch.sentiment, 'correct': train_nominal, } graph_tensors['dev'] = { 'predictions': dev_predictions, 'data_batch': dev_data_batch, 'tokens': dev_minibatch.tokens, 'num_tokens': dev_minibatch.num_tokens, 'minibatch': dev_minibatch, 'bound': dev_bound, 'verified': dev_verified, 'words': dev_words, 'sentiment': dev_minibatch.sentiment, 'correct': dev_nominal, } graph_tensors['test'] = { 'predictions': test_predictions, 'data_batch': test_data_batch, 'tokens': test_minibatch.tokens, 'num_tokens': test_minibatch.num_tokens, 'minibatch': test_minibatch, 'bound': test_bound, 'verified': test_verified, 'words': test_words, 'sentiment': test_minibatch.sentiment, 'correct': test_nominal, } return graph_tensors