def export():
checkpoint_path = FLAGS.checkpoint_path
export_path = FLAGS.export_path
vocab_path = FLAGS.vocab_path
num_steps = FLAGS.num_steps
vocab_size = FLAGS.vocab_size
embedding_size = FLAGS.embedding_size
hidden_size = FLAGS.hidden_size
keep_prob = FLAGS.keep_prob
num_layers = FLAGS.num_layers
num_classes = FLAGS.num_classes
prop_limit = FLAGS.prop_limit
# split 1-D String dense Tensor to words SparseTensor
sentences = tf.placeholder(dtype=tf.string,
shape=[None],
name='input_sentences')
sparse_words = tf.string_split(sentences, delimiter=' ')
# slice SparseTensor
valid_indices = tf.less(sparse_words.indices,
tf.constant([num_steps], dtype=tf.int64))
valid_indices = tf.reshape(
tf.split(valid_indices, [1, 1], axis=1)[1], [-1])
valid_sparse_words = tf.sparse_retain(sparse_words, valid_indices)
excess_indices = tf.greater_equal(sparse_words.indices,
tf.constant([num_steps], dtype=tf.int64))
excess_indices = tf.reshape(
tf.split(excess_indices, [1, 1], axis=1)[1], [-1])
excess_sparse_words = tf.sparse_retain(sparse_words, excess_indices)
# sparse to dense
words = tf.sparse_to_dense(
sparse_indices=valid_sparse_words.indices,
output_shape=[valid_sparse_words.dense_shape[0], num_steps],
sparse_values=valid_sparse_words.values,
default_value='_PAD')
# dict words to token ids
words_table = lookup.index_table_from_file(os.path.join(
vocab_path, 'words_vocab.txt'),
default_value=3)
words_ids = words_table.lookup(words)
# blstm model predict
with tf.variable_scope('model', reuse=None):
logits, _ = ner_model.inference(words_ids,
valid_sparse_words.dense_shape[0],
num_steps,
vocab_size,
embedding_size,
hidden_size,
keep_prob,
num_layers,
num_classes,
is_training=False)
props = tf.nn.softmax(logits)
max_prop_values, max_prop_indices = tf.nn.top_k(props, k=1)
predict_scores = tf.reshape(max_prop_values, shape=[-1, num_steps])
predict_labels_ids = tf.reshape(max_prop_indices, shape=[-1, num_steps])
predict_labels_ids = tf.to_int64(predict_labels_ids)
# replace untrusted prop that less than prop_limit
trusted_prop_flag = tf.greater_equal(
predict_scores, tf.constant(prop_limit, dtype=tf.float32))
replace_prop_labels_ids = tf.to_int64(
tf.fill(tf.shape(predict_labels_ids), 4))
predict_labels_ids = tf.where(trusted_prop_flag, predict_labels_ids,
replace_prop_labels_ids)
# dict token ids to labels
labels_table = lookup.index_to_string_table_from_file(os.path.join(
vocab_path, 'labels_vocab.txt'),
default_value='o')
predict_labels = labels_table.lookup(predict_labels_ids)
# extract real blstm predict label in dense and save to sparse
valid_sparse_predict_labels = tf.SparseTensor(
indices=valid_sparse_words.indices,
values=tf.gather_nd(predict_labels, valid_sparse_words.indices),
dense_shape=valid_sparse_words.dense_shape)
# create excess label SparseTensor with 'O'
excess_sparse_predict_labels = tf.SparseTensor(
indices=excess_sparse_words.indices,
values=tf.fill(tf.shape(excess_sparse_words.values), 'O'),
dense_shape=excess_sparse_words.dense_shape)
# concat SparseTensor
sparse_predict_labels = tf.SparseTensor(
indices=tf.concat(axis=0,
values=[
valid_sparse_predict_labels.indices,
excess_sparse_predict_labels.indices
]),
values=tf.concat(axis=0,
values=[
valid_sparse_predict_labels.values,
excess_sparse_predict_labels.values
]),
dense_shape=excess_sparse_predict_labels.dense_shape)
sparse_predict_labels = tf.sparse_reorder(sparse_predict_labels)
# join SparseTensor to 1-D String dense Tensor
# remain issue, num_split should equal the real size, but here limit to 1
join_labels_list = []
slice_labels_list = tf.sparse_split(sp_input=sparse_predict_labels,
num_split=1,
axis=0)
for slice_labels in slice_labels_list:
slice_labels = slice_labels.values
join_labels = tf.reduce_join(slice_labels,
reduction_indices=0,
separator=' ')
join_labels_list.append(join_labels)
format_predict_labels = tf.stack(join_labels_list)
saver = tf.train.Saver()
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(checkpoint_path)
if ckpt and ckpt.model_checkpoint_path:
print('read model from {}'.format(ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = int(ckpt.model_checkpoint_path.split('-')[-1])
else:
print('No checkpoint file found at %s' % FLAGS.checkpoint_path)
return
# Export inference model.
output_path = os.path.join(export_path, str(global_step))
print 'Exporting trained model to', output_path
builder = tf.saved_model.builder.SavedModelBuilder(output_path)
# Build the signature_def_map.
predict_inputs_tensor_info = tf.saved_model.utils.build_tensor_info(
sentences)
predict_output_tensor_info = tf.saved_model.utils.build_tensor_info(
format_predict_labels)
prediction_signature = tf.saved_model.signature_def_utils.build_signature_def(
inputs={
'input_sentences': predict_inputs_tensor_info,
},
outputs={'classes': predict_output_tensor_info},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={'predict_ner': prediction_signature},
legacy_init_op=legacy_init_op)
builder.save()
print 'Successfully exported model to %s' % export_path
def run_lstm(embedding):
example_batch = input_pipeline(['data/train_examples.tfrecords'],
batch_size=64)
lengths, questions, answers, images = example_batch
final_state = lstm(questions, lengths, embedding)
with tf.variable_scope('mlp'):
weights = tf.get_variable('weights', [512, 28],
initializer=tf.random_normal_initializer())
biases = tf.get_variable('biases', [28],
initializer=tf.zeros_initializer())
out = tf.nn.xw_plus_b(final_state, weights, biases)
with tf.variable_scope('eval'):
predictions = tf.argmax(tf.nn.softmax(out), axis=1)
correct = tf.equal(predictions, answers)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
tf.summary.scalar('accuracy', accuracy)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=out,
labels=answers))
with tf.variable_scope('train'):
train_op = tf.train.AdamOptimizer(1e-4).minimize(loss)
with tf.name_scope('Text_Decode'):
question_table = index_to_string_table_from_file(
vocabulary_file='data/vocabulary.txt', name='Question_Table')
answer_table = index_to_string_table_from_file(
vocabulary_file='data/answers.txt', name='Answer_Table')
question_strings = tf.expand_dims(tf.reduce_join(question_table.lookup(
tf.slice(questions, [0, 0], [5, -1])),
axis=1,
separator=' '),
axis=1)
answer_strings = tf.expand_dims(answer_table.lookup(
tf.slice(answers, [0], [5])),
axis=1)
prediction_strings = tf.expand_dims(answer_table.lookup(
tf.slice(predictions, [0], [5])),
axis=1)
labels = tf.constant(['Question', 'Answer', 'Prediction'],
shape=[1, 3])
qa_table = tf.concat(
[question_strings, answer_strings, prediction_strings], axis=1)
qa_table = tf.concat([labels, qa_table], axis=0)
print(qa_table)
# qa_string = tf.string_join([qa_string, prediction_strings], separator='\r\nPredicted: ')
tf.summary.text('Question', qa_table)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name, var)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
saver = tf.train.Saver()
with tf.Session() as sess:
tf.tables_initializer().run()
sess.run(init_op)
summary_writer = tf.summary.FileWriter('logs/3', graph=sess.graph)
summary_op = tf.summary.merge_all()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(5000001):
if i % 100 == 0:
_, summary, acc = sess.run([train_op, summary_op, accuracy])
summary_writer.add_summary(summary, i)
print('\rStep %d, Accuracy: %f' % (i, acc), end='' * 10)
else:
sess.run(train_op)
print('\rStep %d' % i, end=' ' * 20)
if i % 5000 == 0:
saver.save(sess, 'logs/3/model.ckpt', global_step=i)
coord.request_stop()
coord.join(threads)
def main(embedding):
# img_files = tf.gfile.ListDirectory(img_dir)
# print(img_files)
# img_files = [os.path.join(img_dir, filename) for filename in img_files]
# filename_queue = tf.train.string_input_producer(img_files, shuffle=False, num_epochs=1)
# # img_reader = tf.WholeFileReader()
# # name, img_data = img_reader.read(filename_queue)
# name = filename_queue.dequeue()
id_to_label = {}
with open(os.path.join('id_to_human.json')) as file:
id_to_label = json.load(file)
extract_pretrained_weights('data/resnet_v1_101.ckpt')
example_batch = input_pipeline(['data/train_examples.tfrecords'],
batch_size=32)
lengths, questions, answers, images = example_batch
logits = res_net(images)
# image_cat = tf.argmax(tf.nn.softmax(logits), axis=1)
last_hidden = lstm(questions, lengths, embedding)
print(logits)
print(last_hidden)
output = mlp_layer(logits, last_hidden)
with tf.name_scope('inference'):
predictions = tf.argmax(tf.nn.softmax(output), axis=1)
correct = tf.equal(predictions, answers)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# print(predictions)
# print(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
# print(tf.get_collection(tf.GraphKeys.UPDATE_OPS))
train_op = train_ops(output, answers)
with tf.name_scope('Text_Decode'):
question_table = index_to_string_table_from_file(
vocabulary_file='data/vocabulary.txt', name='Question_Table')
answer_table = index_to_string_table_from_file(
vocabulary_file='data/answers.txt', name='Answer_Table')
question_strings = tf.expand_dims(tf.reduce_join(question_table.lookup(
tf.slice(questions, [0, 0], [5, -1])),
axis=1,
separator=' '),
axis=1)
answer_strings = tf.expand_dims(answer_table.lookup(
tf.slice(answers, [0], [5])),
axis=1)
prediction_strings = tf.expand_dims(answer_table.lookup(
tf.slice(predictions, [0], [5])),
axis=1)
labels = tf.constant(['Question', 'Answer', 'Prediction'],
shape=[1, 3])
qa_table = tf.concat(
[question_strings, answer_strings, prediction_strings], axis=1)
qa_table = tf.concat([labels, qa_table], axis=0)
print(qa_table)
# qa_string = tf.string_join([qa_string, prediction_strings], separator='\r\nPredicted: ')
tf.summary.text('Question', qa_table)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
saver = tf.train.Saver()
with tf.Session() as sess:
tf.tables_initializer().run()
sess.run(init_op)
summary_writer = tf.summary.FileWriter('logs/6', graph=sess.graph)
summary_op = tf.summary.merge_all()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(5000001):
if i % 100 == 0:
_, summary, acc = sess.run([train_op, summary_op, accuracy])
summary_writer.add_summary(summary, i)
print('\rStep %d, Accuracy: %f' % (i, acc))
else:
sess.run(train_op)
print('\rStep %d' % i, end=' ' * 20)
if i % 5000 == 0:
saver.save(sess, 'logs/6/model.ckpt', global_step=i)
coord.request_stop()
coord.join(threads)
def model_fn(self, features, labels, params, mode):
# embedding input and target sequence
vocab_size = params["vocab_size"]
num_units = params["num_units"]
input_keep_prob = params["input_keep_prob"]
layer_size = params["encode_layzer_size"]
in_seq = features["input"]
in_seq_length = features["input_length"]
if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:
out_seq = labels["output_out"]
out_seq_length = labels["output_length"]
else:
out_seq = None
out_seq_length = None
table = lookup.index_table_from_file(params["vocabs_labels_file"],
default_value=0)
features_ids = table.lookup(in_seq)
with tf.variable_scope("embedding"):
embedding = tf.Variable(tf.random.uniform([vocab_size, num_units],
-1.0, 1.0),
dtype=tf.float32,
name='word_embedding')
# embedding = tf.get_variable(name='embed', shape=[vocab_size, num_units])
embed_input = tf.nn.embedding_lookup(params=embedding,
ids=features_ids,
name="embed_input")
# encode and decode
bi_layer_size = int(layer_size / 2)
encoder_cell_fw = self.get_layered_cell(bi_layer_size, num_units,
input_keep_prob)
encoder_cell_bw = self.get_layered_cell(bi_layer_size, num_units,
input_keep_prob)
# sequence_length 应该是序列真实的长度
bi_encoder_output, bi_encoder_state = tf.nn.bidirectional_dynamic_rnn(
cell_fw=encoder_cell_fw,
cell_bw=encoder_cell_bw,
inputs=embed_input,
sequence_length=in_seq_length,
dtype=embed_input.dtype,
time_major=False)
# concat encode output and state
encoder_output = tf.concat(bi_encoder_output, -1)
encoder_state = []
for layers_id in range(bi_layer_size):
encoder_state.append(bi_encoder_state[0][layers_id])
encoder_state.append(bi_encoder_state[1][layers_id])
encoder_state = tuple(encoder_state)
decoder_cell = self.attention_decoder_cell(encoder_output,
in_seq_length,
num_units, layer_size,
input_keep_prob)
batch_size = tf.shape(in_seq_length)[0]
init_state = decoder_cell.zero_state(
batch_size, tf.float32).clone(cell_state=encoder_state)
# str2int
labels_ids = None
if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:
labels_ids = table.lookup(out_seq)
embed_target = tf.nn.embedding_lookup(embedding,
labels_ids,
name="embed_target")
helper = tf.contrib.seq2seq.TrainingHelper(embed_target,
out_seq_length,
time_major=False)
else:
# TODO: start tokens and end tokens are hard code
helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(
embedding, tf.fill([batch_size], 0), 1)
predict_output_layer = core.Dense(vocab_size, use_bias=False)
basic_decoder = tf.contrib.seq2seq.BasicDecoder(
cell=decoder_cell,
helper=helper,
initial_state=init_state,
output_layer=predict_output_layer)
# 执行动态解码 (final_outputs, final_state, final_sequence_lengths)
final_outputs, final_state, final_sequence_lengths = tf.contrib.seq2seq.dynamic_decode(
decoder=basic_decoder, maximum_iterations=50)
loss = None
train_op = None
predictions = None
if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:
outputs = final_outputs.rnn_output
loss = self.compute_loss(outputs, labels_ids, out_seq_length)
trainable_params = tf.trainable_variables()
global_step = tf.train.get_global_step()
clipped_gradients, _ = tf.clip_by_global_norm(
tf.gradients(loss, trainable_params), 0.8)
train_op = tf.train.AdamOptimizer(
learning_rate=0.001).apply_gradients(
zip(clipped_gradients, trainable_params),
global_step=global_step)
# train_op = tf.train.AdamOptimizer(learning_rate=0.01).minimize(loss)
# tf.summary.scalar('loss', loss)
tf.add_to_collection("loss", loss)
else:
outputs = final_outputs.sample_id
predict_ids = tf.cast(outputs, dtype=tf.int64)
mask = tf.sequence_mask(in_seq_length,
maxlen=tf.shape(predict_ids)[1])
real_ids = predict_ids * tf.cast(mask, dtype=tf.int64)
index_to_string = lookup.index_to_string_table_from_file(
params["vocabs_labels_file"])
predict_labels = index_to_string.lookup(real_ids)
predictions = {
"predict_labels": predict_labels,
"predict_ids": real_ids,
}
tf.add_to_collection("predictions", predictions)
return tf.estimator.EstimatorSpec(mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op)
def testModel(self):
# 数据输入
dataset_f = tf.data.TextLineDataset(filenames=FEATURE_FILE)
dataset_l = tf.data.TextLineDataset(filenames=LABEL_FILE)
dataset = tf.data.Dataset.zip((dataset_f, dataset_l))
dataset = dataset.map(lambda x, y:
(tf.string_split([x], delimiter=",").values,
tf.string_split([y], delimiter=",").values))
dataset = dataset.shuffle(1000)
dataset = dataset.map(lambda x, y: (x, tf.size(x), y))
dataset = dataset.padded_batch(
batch_size=2,
padded_shapes=([
None,
], [], [None]),
padding_values=(tf.constant("<UNK>", dtype=tf.string), 0,
tf.constant("O", dtype=tf.string)))
iterator = dataset.make_initializable_iterator()
tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS,
iterator.initializer)
feature, feature_length, label = iterator.get_next()
# 通过字典获得索引
words2idx = lookup_op.index_table_from_file(VOCAB_FEATURE)
words_ids = words2idx.lookup(feature)
# 编码
with tf.variable_scope("embedding_test", reuse=tf.AUTO_REUSE):
embedding_variable = tf.get_variable("embedding_variable",
shape=[18, 256],
dtype=tf.double)
embedding = tf.nn.embedding_lookup(embedding_variable, words_ids)
embedding_outputs = tf.layers.dropout(inputs=embedding,
rate=0.5,
training=True)
inputs = embedding_outputs
# 模型
with tf.variable_scope("lstm_model", reuse=tf.AUTO_REUSE):
cells = [
tf.nn.rnn_cell.LSTMCell(256),
tf.nn.rnn_cell.LSTMCell(256)
]
cell = tf.nn.rnn_cell.MultiRNNCell(cells=cells)
outputs, state = tf.nn.dynamic_rnn(cell=cell,
inputs=inputs,
sequence_length=feature_length,
dtype=inputs.dtype)
# 输出层
logits = tf.layers.dense(inputs=outputs, units=5)
# 解码
transition = tf.get_variable(name="transition",
shape=[5, 5],
dtype=tf.double)
decode_ids, best_score = tf.contrib.crf.crf_decode(
logits, transition, feature_length)
# 通过索引获得标签
idx2tags = lookup_op.index_to_string_table_from_file(VOCAB_LABEL,
default_value="O")
tags = idx2tags.lookup(tf.cast(decode_ids, dtype=tf.int64))
# 计算损失
tags2idx = lookup_op.index_table_from_file(VOCAB_LABEL)
actual_ids = tags2idx.lookup(label)
log_likeilhood, _ = tf.contrib.crf.crf_log_likelihood(
inputs=tf.cast(logits, dtype=tf.float32),
tag_indices=actual_ids,
sequence_lengths=feature_length)
loss = tf.reduce_mean(-log_likeilhood)
with self.test_session() as sess:
sess.run(tf.get_collection(tf.GraphKeys.TABLE_INITIALIZERS))
sess.run(tf.global_variables_initializer())
# print(sess.run(feature))
# print(sess.run(feature_length))
# print(sess.run(label))
# print(sess.run(words2idx))
# print(sess.run(words_ids))
# print(sess.run(embedding_variable))
# print(sess.run(tf.shape(embedding_variable)))
# print(sess.run(embedding))
# print(sess.run(tf.shape(embedding)))
# print(sess.run(embedding_outputs))
# print(sess.run(tf.shape(embedding_outputs)))
# print(sess.run(outputs))
# print(sess.run(tf.shape(outputs)))
# print(sess.run(tf.shape(state[0])))
# print(sess.run(logits))
# print(sess.run(tf.shape(logits)))
# print(sess.run(decode_tags))
# print(sess.run(tags))
# print(sess.run(actual_ids))
# print(sess.run(log_likeilhood))
print(sess.run(loss))
return
def tensor_predict(words_list):
num_classes = FLAGS.num_classes
num_layers = FLAGS.num_layers
num_steps = FLAGS.num_steps
embedding_size = FLAGS.embedding_size
hidden_size = FLAGS.hidden_size
keep_prob = FLAGS.keep_prob
vocab_size = FLAGS.vocab_size
vocab_path = FLAGS.vocab_path
prop_limit = FLAGS.prop_limit
checkpoint_path = FLAGS.checkpoint_path
# split 1-D String dense Tensor to words SparseTensor
sentences = tf.placeholder(dtype=tf.string,
shape=[None],
name='input_sentences')
sparse_words = tf.string_split(sentences, delimiter=' ')
# slice SparseTensor
valid_indices = tf.less(sparse_words.indices,
tf.constant([num_steps], dtype=tf.int64))
valid_indices = tf.reshape(
tf.split(valid_indices, [1, 1], axis=1)[1], [-1])
valid_sparse_words = tf.sparse_retain(sparse_words, valid_indices)
excess_indices = tf.greater_equal(sparse_words.indices,
tf.constant([num_steps], dtype=tf.int64))
excess_indices = tf.reshape(
tf.split(excess_indices, [1, 1], axis=1)[1], [-1])
excess_sparse_words = tf.sparse_retain(sparse_words, excess_indices)
# sparse to dense
words = tf.sparse_to_dense(
sparse_indices=valid_sparse_words.indices,
output_shape=[valid_sparse_words.dense_shape[0], num_steps],
sparse_values=valid_sparse_words.values,
default_value='_PAD')
# dict words to token ids
# with open(os.path.join(vocab_path, 'words_vocab.txt'), 'r') as data_file:
# words_table_list = [line.strip() for line in data_file if line.strip()]
# words_table_tensor = tf.constant(words_table_list, dtype=tf.string)
# words_table = lookup.index_table_from_tensor(mapping=words_table_tensor, default_value=3)
words_table = lookup.index_table_from_file(os.path.join(
vocab_path, 'words_vocab.txt'),
default_value=3)
words_ids = words_table.lookup(words)
# blstm model predict
with tf.variable_scope('model', reuse=None):
logits, _ = model.inference(words_ids,
valid_sparse_words.dense_shape[0],
num_steps,
vocab_size,
embedding_size,
hidden_size,
keep_prob,
num_layers,
num_classes,
is_training=False)
# using softmax
# props = tf.nn.softmax(logits)
# max_prop_values, max_prop_indices = tf.nn.top_k(props, k=1)
# predict_scores = tf.reshape(max_prop_values, shape=[-1, num_steps])
# predict_labels_ids = tf.reshape(max_prop_indices, shape=[-1, num_steps])
# predict_labels_ids = tf.to_int64(predict_labels_ids)
# using crf
logits = tf.reshape(logits, shape=[-1, num_steps, num_classes])
transition_params = tf.get_variable("transitions",
[num_classes, num_classes])
sequence_length = tf.constant(num_steps,
shape=[logits.get_shape()[0]],
dtype=tf.int64)
predict_labels_ids, _ = crf_utils.crf_decode(logits, transition_params,
sequence_length)
predict_labels_ids = tf.to_int64(predict_labels_ids)
predict_scores = tf.constant(1.0,
shape=predict_labels_ids.get_shape(),
dtype=tf.float32)
# replace untrusted prop that less than prop_limit
trusted_prop_flag = tf.greater_equal(
predict_scores, tf.constant(prop_limit, dtype=tf.float32))
replace_prop_labels_ids = tf.to_int64(
tf.fill(tf.shape(predict_labels_ids), 4))
predict_labels_ids = tf.where(trusted_prop_flag, predict_labels_ids,
replace_prop_labels_ids)
# dict token ids to labels
# with open(os.path.join(vocab_path, 'labels_vocab.txt'), 'r') as data_file:
# labels_table_list = [line.strip() for line in data_file if line.strip()]
# labels_table_tensor = tf.constant(labels_table_list, dtype=tf.string)
# labels_table = lookup.index_to_string_table_from_tensor(mapping=labels_table_tensor, default_value='O')
labels_table = lookup.index_to_string_table_from_file(os.path.join(
vocab_path, 'labels_vocab.txt'),
default_value='O')
predict_labels = labels_table.lookup(predict_labels_ids)
# extract real blstm predict label in dense and save to sparse
valid_sparse_predict_labels = tf.SparseTensor(
indices=valid_sparse_words.indices,
values=tf.gather_nd(predict_labels, valid_sparse_words.indices),
dense_shape=valid_sparse_words.dense_shape)
# create excess label SparseTensor with 'O'
excess_sparse_predict_labels = tf.SparseTensor(
indices=excess_sparse_words.indices,
values=tf.fill(tf.shape(excess_sparse_words.values), 'O'),
dense_shape=excess_sparse_words.dense_shape)
# concat SparseTensor
sparse_predict_labels = tf.SparseTensor(
indices=tf.concat(axis=0,
values=[
valid_sparse_predict_labels.indices,
excess_sparse_predict_labels.indices
]),
values=tf.concat(axis=0,
values=[
valid_sparse_predict_labels.values,
excess_sparse_predict_labels.values
]),
dense_shape=excess_sparse_predict_labels.dense_shape)
sparse_predict_labels = tf.sparse_reorder(sparse_predict_labels)
# join SparseTensor to 1-D String dense Tensor
# remain issue, num_split should equal the real size, but here limit to 1
join_labels_list = []
slice_labels_list = tf.sparse_split(sp_input=sparse_predict_labels,
num_split=1,
axis=0)
for slice_labels in slice_labels_list:
slice_labels = slice_labels.values
join_labels = tf.reduce_join(slice_labels,
reduction_indices=0,
separator=' ')
join_labels_list.append(join_labels)
format_predict_labels = tf.stack(join_labels_list, name='predict_labels')
saver = tf.train.Saver()
tables_init_op = tf.tables_initializer()
with tf.Session() as sess:
sess.run(tables_init_op)
ckpt = tf.train.get_checkpoint_state(checkpoint_path)
if ckpt and ckpt.model_checkpoint_path:
print('read model from {}'.format(ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found at %s' % checkpoint_path)
return
# crf tensor
predict_labels_list = sess.run(format_predict_labels,
feed_dict={sentences: words_list})
# save graph into .pb file
graph = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, ["init_all_tables", "predict_labels"])
tf.train.write_graph(graph, '.', 'ner_graph.pb', as_text=False)
return predict_labels_list
def vocabulary_lookup_reverse(self):
# build a table for ids2word
return lookup.index_to_string_table_from_file(
vocabulary_file=self.vocabulary_file,
vocab_size=self.size - self.num_oov_buckets,
default_value=UNKNOWN_TOKEN)
def _build_graph_with_datasets(self, train_minibatch, dev_minibatch,
test_minibatch, num_classes):
"""Returns the training/evaluation ops."""
self.keep_prob = 1. # Using literal 1 (not placeholder) skips dropout op.
self.sentence_representer._keep_prob = 1. # pylint:disable=protected-access
# Build the graph as per the base class.
(train_joint_rep, _) = self.add_representation(train_minibatch)
(train_accuracy, loss,
opt_step) = self.add_train_ops(num_classes, train_joint_rep,
train_minibatch)
dev_num_correct = self.add_dev_eval_ops(dev_minibatch)
test_num_correct = self.add_dev_eval_ops(test_minibatch)
graph_tensors = {
'loss': loss,
'train_op': opt_step,
'train_accuracy': train_accuracy,
'dev_num_correct': dev_num_correct,
'test_num_correct': test_num_correct,
'keep_prob': self.keep_prob
}
vocab_table = self.embed_pad.vocab_table
vocab_size = self.embed_pad.vocab_size
verifiable_loss_ratio = tf.constant(
self.config['verifiable_loss_ratio'],
dtype=tf.float32,
name='verifiable_loss_ratio')
self.delta = tf.constant(self.config['delta'],
dtype=tf.float32,
name='delta')
lookup_token = tf.placeholder(tf.string,
shape=None,
name='lookup_token')
indices = vocab_table.lookup(lookup_token)
self.vocab_list = contrib_lookup.index_to_string_table_from_file(
self.config['vocab_filename_pad'])
lookup_token_index = tf.placeholder(tf.int64,
shape=None,
name='lookup_token_index')
lookup_token_string = self.vocab_list.lookup(lookup_token_index)
synonym_values = tf.placeholder(tf.int64,
shape=[None, None],
name='synonym_values')
synonym_counts = tf.placeholder(tf.int64,
shape=[None],
name='synonym_counts')
train_perturbation = self.create_perturbation_ops(
train_minibatch, synonym_values, vocab_table)
train_data_batch, _, _ = self.preprocess_databatch(
train_minibatch, vocab_table, train_perturbation)
train_words = self.vocab_list.lookup(train_data_batch.input_tokens)
# [num_targets x batchsize]
verifiable_obj = self.add_verifiable_objective(train_minibatch,
vocab_table,
train_perturbation,
stop_gradient=False)
train_nominal = self.run_classification(
train_data_batch.embedded_inputs, train_data_batch.sentiment,
train_data_batch.length)
train_bound, train_verified = self.compute_verifiable_verified(
verifiable_obj)
verifiable_loss = self.compute_verifiable_loss(
verifiable_obj, train_minibatch.sentiment)
if (self.config['verifiable_loss_ratio']) > 1.0:
raise ValueError('Loss ratios sum up to more than 1.0')
total_loss = (1 - verifiable_loss_ratio) * graph_tensors['loss']
if self.config['verifiable_loss_ratio'] != 0:
total_loss += verifiable_loss_ratio * verifiable_loss
# Attack on dev/test set.
dev_perturbation = self.create_perturbation_ops(
dev_minibatch, synonym_values, vocab_table)
# [num_targets x batchsize]
dev_verifiable_obj = self.add_verifiable_objective(dev_minibatch,
vocab_table,
dev_perturbation,
stop_gradient=True)
dev_bound, dev_verified = self.compute_verifiable_verified(
dev_verifiable_obj)
dev_data_batch, _, _ = self.preprocess_databatch(
dev_minibatch, vocab_table, dev_perturbation)
test_perturbation = self.create_perturbation_ops(
test_minibatch, synonym_values, vocab_table)
# [num_targets x batchsize]
test_verifiable_obj = self.add_verifiable_objective(test_minibatch,
vocab_table,
test_perturbation,
stop_gradient=True)
test_bound, test_verified = self.compute_verifiable_verified(
test_verifiable_obj)
test_data_batch, _, _ = self.preprocess_databatch(
test_minibatch, vocab_table, test_perturbation)
dev_words = self.vocab_list.lookup(dev_data_batch.input_tokens)
test_words = self.vocab_list.lookup(test_data_batch.input_tokens)
dev_nominal = self.run_classification(dev_data_batch.embedded_inputs,
dev_data_batch.sentiment,
dev_data_batch.length)
test_nominal = self.run_classification(test_data_batch.embedded_inputs,
test_data_batch.sentiment,
test_data_batch.length)
dev_predictions = self.run_prediction(dev_data_batch.embedded_inputs,
dev_data_batch.length)
test_predictions = self.run_prediction(test_data_batch.embedded_inputs,
test_data_batch.length)
with tf.control_dependencies(
[train_verified, test_verified, dev_verified]):
opt_step = self._add_optimize_op(total_loss)
graph_tensors['total_loss'] = total_loss
graph_tensors['verifiable_loss'] = verifiable_loss
graph_tensors['train_op'] = opt_step
graph_tensors['indices'] = indices
graph_tensors['lookup_token_index'] = lookup_token_index
graph_tensors['lookup_token_string'] = lookup_token_string
graph_tensors['lookup_token'] = lookup_token
graph_tensors['vocab_size'] = vocab_size
graph_tensors['synonym_values'] = synonym_values
graph_tensors['synonym_counts'] = synonym_counts
graph_tensors['verifiable_loss_ratio'] = verifiable_loss_ratio
graph_tensors['delta'] = self.delta
graph_tensors['train'] = {
'bound': train_bound,
'verified': train_verified,
'words': train_words,
'sentiment': train_minibatch.sentiment,
'correct': train_nominal,
}
graph_tensors['dev'] = {
'predictions': dev_predictions,
'data_batch': dev_data_batch,
'tokens': dev_minibatch.tokens,
'num_tokens': dev_minibatch.num_tokens,
'minibatch': dev_minibatch,
'bound': dev_bound,
'verified': dev_verified,
'words': dev_words,
'sentiment': dev_minibatch.sentiment,
'correct': dev_nominal,
}
graph_tensors['test'] = {
'predictions': test_predictions,
'data_batch': test_data_batch,
'tokens': test_minibatch.tokens,
'num_tokens': test_minibatch.num_tokens,
'minibatch': test_minibatch,
'bound': test_bound,
'verified': test_verified,
'words': test_words,
'sentiment': test_minibatch.sentiment,
'correct': test_nominal,
}
return graph_tensors