def export_pb_model():
vocabulary_word2index, vocabulary_index2word = create_term(
FLAGS.term_index_path)
vocabulary_char2index, vocabulary_index2char = create_term(
FLAGS.char_index_path)
vocab_size = len(vocabulary_word2index)
char_size = len(vocabulary_char2index)
with tf.Graph().as_default():
model = BiGRUAttention(FLAGS.loss_number,
FLAGS.num_count,
FLAGS.num_classes,
FLAGS.learning_rate,
FLAGS.batch_size,
FLAGS.decay_steps,
FLAGS.decay_rate,
FLAGS.sequence_length,
FLAGS.chars_length,
FLAGS.num_sentences,
vocab_size,
char_size,
FLAGS.embed_size,
FLAGS.hidden_size,
FLAGS.is_training,
multi_label_flag=FLAGS.multi_label_flag,
char_attention_flag=FLAGS.char_attention_flag,
count_flag=FLAGS.count_flag)
model_signature = signature_def_utils.build_signature_def(
inputs={
"terms_ids":
utils.build_tensor_info(model.input_x),
"keep_prob_hidden":
utils.build_tensor_info(model.dropout_keep_prob)
},
outputs={"prediction": utils.build_tensor_info(model.predictions)},
method_name=signature_constants.CLASSIFY_METHOD_NAME)
session_conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess = tf.Session(config=session_conf)
saver = tf.train.Saver()
#saver.restore(sess,_ckpt+'model.ckpt-6')
print(tf.train.latest_checkpoint(FLAGS.ckpt))
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt))
builder = saved_model_builder.SavedModelBuilder(FLAGS.pb_path)
builder.add_meta_graph_and_variables(sess, [tag_constants.SERVING],
clear_devices=True,
signature_def_map={
'cat_han_model_signature':
model_signature,
})
builder.save()
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
if not FLAGS.do_train and not FLAGS.do_eval and not FLAGS.do_predict:
raise ValueError(
"At least one of do_train, do_eval or do_predict must be True")
tpu_cluster_resolver = None
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
run_config = tf.contrib.tpu.RunConfig(
cluster=tpu_cluster_resolver,
master=None,
model_dir=FLAGS.output_dir,
save_checkpoints_steps=FLAGS.save_checkpoints_steps)
#1. load vocabulary
vocabulary_word2index, vocabulary_index2word = create_term(
FLAGS.term_index_path)
vocab_size = len(vocabulary_index2word)
print("vocab_size:", vocab_size)
model_fn = model_fn_builder(vocab_size=vocab_size)
estimator = tf.contrib.tpu.TPUEstimator(
use_tpu=None,
model_fn=model_fn,
config=run_config,
train_batch_size=FLAGS.batch_size,
eval_batch_size=FLAGS.batch_size,
predict_batch_size=FLAGS.batch_size)
if FLAGS.do_train:
num_train_steps = int(
FLAGS.train_sample_num / FLAGS.batch_size) * FLAGS.num_epochs
print("*****all steps **************", num_train_steps)
train_input_fn = file_based_input_fn_builder(
num_cpu_threads=FLAGS.num_cpu_threads,
input_file=train_sample_file,
batch_size=FLAGS.batch_size,
seq_length=FLAGS.sequence_length,
first_length=FLAGS.first_length,
second_length=FLAGS.second_length,
third_length=FLAGS.third_length,
is_training=True,
drop_remainder=True)
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
def __init__(self):
self.vocabulary_word2index, self.vocabulary_index2word = create_term(
FLAGS.term_index_path)
self.vocab_size = len(self.vocabulary_word2index)
self.char_size = 100
self.vocabulary_word2index_label, self.vocabulary_index2word_label = create_label(
FLAGS.label_index_path)
self.label_name = load_cid(FLAGS.label_index_path)
FLAGS.num_classes = len(self.vocabulary_word2index_label)
with tf.Graph().as_default():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
# 4.Instantiate Model
self.model = BiGRUAttention(
FLAGS.loss_number,
FLAGS.num_count,
FLAGS.num_classes,
FLAGS.learning_rate,
FLAGS.batch_size,
FLAGS.decay_steps,
FLAGS.decay_rate,
FLAGS.sequence_length,
FLAGS.chars_length,
FLAGS.num_sentences,
self.vocab_size,
self.char_size,
FLAGS.embed_size,
FLAGS.hidden_size,
FLAGS.is_training,
multi_label_flag=FLAGS.multi_label_flag,
char_attention_flag=FLAGS.char_attention_flag,
count_flag=FLAGS.count_flag)
self.saver = tf.train.Saver()
if os.path.exists(FLAGS.ckpt_dir + "checkpoint"):
print("Restoring Variables from Checkpoint")
self.saver.restore(self.sess,
tf.train.latest_checkpoint(FLAGS.ckpt_dir))
else:
print("Can't find the checkpoint.going to stop")
def main(_):
print("embed_size:", FLAGS.embed_size)
print("hidden_size:", FLAGS.hidden_size)
print("cid3_num:", FLAGS.num_classes)
print("train_sample_num:", FLAGS.train_sample_num)
print("dev_sample_num:", FLAGS.dev_sample_num)
print("ckpt_dir:", FLAGS.ckpt_dir)
#1. load vocabulary
vocabulary_word2index, vocabulary_index2word = create_term(
FLAGS.term_index_path)
vocab_size = len(vocabulary_index2word)
char_size = FLAGS.char_size
print("vocab_size:", vocab_size)
if FLAGS.char_attention_flag:
vocabulary_char2index, vocabulary_index2char = create_term(
FLAGS.char_index_path)
char_size = len(vocabulary_index2char)
print("char_size:", char_size)
#2.create session.
with tf.Graph().as_default():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
model = BiGRUAttention(FLAGS.loss_number,
FLAGS.num_count,
FLAGS.num_classes,
FLAGS.learning_rate,
FLAGS.batch_size,
FLAGS.decay_steps,
FLAGS.decay_rate,
FLAGS.sequence_length,
FLAGS.chars_length,
FLAGS.num_sentences,
vocab_size,
char_size,
FLAGS.embed_size,
FLAGS.hidden_size,
FLAGS.is_training,
multi_label_flag=FLAGS.multi_label_flag,
char_attention_flag=FLAGS.char_attention_flag,
count_flag=FLAGS.count_flag)
train_batcher = batch_read_tfrecord.SegBatcher(
FLAGS.train_sample_file,
FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
dev_batcher = batch_read_tfrecord.SegBatcher(
FLAGS.dev_sample_file,
FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
global_init_op = tf.global_variables_initializer()
local_init_op = tf.local_variables_initializer()
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
#tensorboard
train_writer = tf.summary.FileWriter(FLAGS.summary_dir + 'train/',
sess.graph)
dev_writer = tf.summary.FileWriter(FLAGS.summary_dir + 'dev/')
sess.run(global_init_op)
sess.run(local_init_op)
if os.path.exists(FLAGS.ckpt_dir + "checkpoint"):
print("Restoring Variables from Checkpoint")
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir))
else:
print('Initializing Variables')
if FLAGS.use_embedding:
assign_pretrained_word_embedding(
sess,
vocabulary_index2word,
vocab_size,
model,
word2vec_model_path=FLAGS.word2vec_model_path)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
curr_epoch = sess.run(model.epoch_step)
#3.feed data & training
train_sample_num = FLAGS.train_sample_num
dev_sample_num = FLAGS.dev_sample_num
best_eval_f1 = 0
for epoch in range(curr_epoch, FLAGS.num_epochs):
loss, acc, recall, precision, f1, counter = 0.0, 0.0, 0.0, 0.0, 0.0, 0
eval_loss, eval_acc, eval_recall, eval_precision, eval_f1, counter_dev = 0.0, 0.0, 0.0, 0.0, 0.0, 0
train_example_num = 0
dev_example_num = 0
while train_example_num < train_sample_num:
try:
train_batch_data = sess.run(
train_batcher.next_batch_op)
trainX, trainXChar, trainY = train_batch_data
if len(trainX) != FLAGS.batch_size:
continue
#trainX = pad_sequences(trainX, maxlen=FLAGS.sequence_length, value=0.)
trainY = common.get_one_hot_label(
trainY, FLAGS.num_classes)
if FLAGS.char_attention_flag:
#trainXChar = pad_sequences(trainXChar, maxlen=FLAGS.chars_length, value=0.)
feed_dict = {
model.input_x: trainX,
model.input_char: trainXChar,
model.dropout_keep_prob: 0.5
}
else:
feed_dict = {
model.input_x: trainX,
model.dropout_keep_prob: 0.5
}
if not FLAGS.multi_label_flag:
feed_dict[model.input_y] = trainY
else:
feed_dict[model.input_y_multilabel] = trainY
summary_merge, global_step, curr_loss, curr_acc, curr_f1, curr_recall, curr_precision, _ = sess.run(
[
model.summary_merge, model.global_step,
model.loss_val, model.accuracy, model.f1,
model.recall, model.precision, model.train_op
], feed_dict)
train_writer.add_summary(summary_merge, global_step)
loss, counter, acc, recall, precision, f1 = loss + curr_loss, counter + 1, acc + curr_acc, recall + curr_recall, precision + curr_precision, f1 + curr_f1
train_example_num += FLAGS.batch_size
if counter % 2000 == 0:
print(
"Epoch %d\tBatch %d\tTrain Loss:%.3f\tTrain Accuracy:%.3f\tTrain precision:%.3f\tTrain recall:%.3f\tTrain f1:%.3f"
%
(epoch, counter, loss / float(counter), acc /
float(counter), precision / float(counter),
recall / float(counter), f1 / float(counter)))
except tf.errors.OutOfRangeError:
print("Done Training")
break
##VALIDATION VALIDATION VALIDATION PART######################################################################################################
while dev_example_num < dev_sample_num:
counter_dev += 1
try:
dev_batch_data = sess.run(dev_batcher.next_batch_op)
testX, testXChar, testY = dev_batch_data
if len(testX) != FLAGS.batch_size:
continue
#testX = pad_sequences(testX, maxlen=FLAGS.sequence_length, value=0.)
testY = common.get_one_hot_label(
testY, FLAGS.num_classes)
if FLAGS.char_attention_flag:
testXChar = pad_sequences(
testXChar, maxlen=FLAGS.chars_length, value=0.)
else:
testXChar = []
cur_eval_loss, cur_eval_acc, cur_eval_precision, cur_eval_recall, cur_eval_f1 = do_eval(
dev_writer, sess, model, testX, testXChar, testY)
eval_loss, eval_acc, eval_recall, eval_precision, eval_f1 = eval_loss + cur_eval_loss, eval_acc + cur_eval_acc, eval_recall + cur_eval_recall, eval_precision + cur_eval_precision, eval_f1 + cur_eval_f1
dev_example_num += FLAGS.batch_size
if counter_dev % FLAGS.validate_step == 0:
print(
"Epoch %d \tBatch %d\tValidation Loss:%.3f\tValidation Accuracy: %.3f\tValidation precision:%.3f\tValidation recall:%.3f\tValidation f1:%.3f\t"
% (epoch, counter_dev,
eval_loss / float(counter_dev),
eval_acc / float(counter_dev),
eval_precision / float(counter_dev),
eval_recall / float(counter_dev),
eval_f1 / float(counter_dev)))
if eval_f1 > best_eval_f1:
best_eval_f1 = eval_f1
print("Validation best f1: %f" % best_eval_f1)
except tf.errors.OutOfRangeError:
print("Done test")
break
##VALIDATION VALIDATION VALIDATION PART######################################################################################################
#epoch increment
print("going to increment epoch counter....")
sess.run(model.epoch_increment)
# 4.validation
print(epoch, FLAGS.validate_every,
(epoch % FLAGS.validate_every == 0))
if epoch % FLAGS.validate_every == 0:
#save model to checkpoint
if not os.path.exists(FLAGS.ckpt_dir):
os.makedirs(FLAGS.ckpt_dir)
save_path = FLAGS.ckpt_dir + "model.ckpt"
saver.save(sess, save_path, global_step=epoch)
coord.request_stop()
coord.join(threads)
sess.close()
def main(_):
# 1.load data with vocabulary of words and labels
vocabulary_word2index, vocabulary_index2word = create_term(
FLAGS.term_index_path)
vocabulary_char2index, vocabulary_index2char = create_term(
FLAGS.char_index_path)
vocab_size = len(vocabulary_word2index)
char_size = len(vocabulary_char2index)
vocabulary_word2index_label, vocabulary_index2word_label = create_label(
FLAGS.label_index_path)
testX, testXChar, lines = load_test(FLAGS.predict_target_file,
vocabulary_word2index,
vocabulary_char2index)
FLAGS.num_classes = len(vocabulary_word2index_label)
print("start padding....")
#testX = pad_sequences(testX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length
if FLAGS.char_attention_flag:
testXChar = pad_sequences(testXChar,
maxlen=FLAGS.chars_length,
value=0.)
print("end padding...")
# 2.create session.
with tf.Graph().as_default():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
model = BiGRUAttention(
FLAGS.loss_number,
FLAGS.num_count,
FLAGS.num_classes,
FLAGS.learning_rate,
FLAGS.batch_size,
FLAGS.decay_steps,
FLAGS.decay_rate,
FLAGS.sequence_length,
FLAGS.chars_length,
FLAGS.num_sentences,
vocab_size,
char_size,
FLAGS.embed_size,
FLAGS.hidden_size,
FLAGS.is_training,
multi_label_flag=FLAGS.multi_label_flag,
char_attention_flag=FLAGS.char_attention_flag,
count_flag=FLAGS.count_flag)
saver = tf.train.Saver()
if os.path.exists(FLAGS.ckpt_dir + "checkpoint"):
print("Restoring Variables from Checkpoint")
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir))
#saver.restore(sess,FLAGS.ckpt_dir+'model.ckpt-2')
else:
print("Can't find the checkpoint.going to stop")
return
# 5.feed data, to get logits
number_of_training_data = len(testX)
print("number_of_training_data:", number_of_training_data)
predict_target_file_f = codecs.open(FLAGS.predict_source_file, 'a',
'utf8')
label_name = load_cid(FLAGS.label_index_path)
t0 = time.time()
for start, end in zip(
range(0, number_of_training_data, FLAGS.batch_size),
range(FLAGS.batch_size, number_of_training_data + 1,
FLAGS.batch_size)):
if FLAGS.char_attention_flag:
predictions = sess.run(model.predictions,
feed_dict={
model.input_x:
testX[start:end],
model.input_char:
testXChar[start:end],
model.dropout_keep_prob:
1
})
else:
attention, predictions = sess.run(
[model.attention, model.predictions],
feed_dict={
model.input_x: testX[start:end],
model.dropout_keep_prob: 1
})
lines_sublist = lines[start:end]
get_label_using_logits_batch(lines_sublist, attention,
predictions,
vocabulary_index2word_label,
predict_target_file_f,
FLAGS.threshold, label_name)
t1 = time.time()
print("all running time: %s " % str(t1 - t0))
predict_target_file_f.close()
return model_fn
def serving_input_receiver_fn():
input_x = tf.placeholder(dtype=tf.int32, shape=[None, None], name='input_x')
receiver_tensors = {'input_x': input_x}
features = {'input_x': input_x}
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
if __name__=='__main__':
#1. load vocabulary
vocabulary_word2index, vocabulary_index2word= create_term(FLAGS.term_index_path)
vocab_size = len(vocabulary_index2word)
print("vocab_size:",vocab_size)
cp_file = tf.train.latest_checkpoint(FLAGS.output_dir)
model_fn = model_fn_builder(
vocab_size=vocab_size)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 1.0
config.log_device_placement = False
batch_size = 1
export_dir = FLAGS.output_dir
estimator = tf.estimator.Estimator(model_fn=model_fn, model_dir=FLAGS.output_dir, config=RunConfig(session_config=config),
params={'batch_size': batch_size})
estimator.export_saved_model(export_dir, serving_input_receiver_fn, checkpoint_path=cp_file)
def predict():
# 1.load data with vocabulary of words and labels
vocabulary_word2index, vocabulary_index2word = create_term(
FLAGS.term_index_path)
vocab_size = len(vocabulary_word2index)
testX, lines = load_test(FLAGS.predict_target_file, vocabulary_word2index)
f = open(FLAGS.predict_source_file, 'w')
# for tags index
#tags_index2word = load_tag(FLAGS.tags_index_path)
# id2name
cid2name = load_cid(FLAGS.label_name_path)
brand2name = load_cid(FLAGS.brand_name_path)
product2name = load_cid(FLAGS.product_name_path)
# id2index
cid2index, index2cid = create_label(FLAGS.label_index_path)
product2index, index2product = create_label(FLAGS.product_index_path)
brand2index, index2brand = create_label(FLAGS.brand_index_path)
#path
subdirs = [
x for x in Path(FLAGS.ckpt_dir).iterdir()
if x.is_dir() and 'temp' not in str(x)
]
model_pb = str(sorted(subdirs)[-1])
predict_fn = predictor.from_saved_model(model_pb)
#evl
for i, x in enumerate(testX):
print(x)
feed_dict = {'input_x': [x]}
result = predict_fn(feed_dict)
batch_predictions = result['predictions']
batch_product_predictions = result['product_predictions']
batch_brand_predictions = result['brand_predictions']
for predictions, product_predictions, brand_predictions in zip(
batch_predictions, batch_product_predictions,
batch_brand_predictions):
predictions_sorted = sorted(predictions, reverse=True)
product_predictions_sorted = sorted(product_predictions,
reverse=True)
brand_predictions_sorted = sorted(brand_predictions, reverse=True)
index_sorted = np.argsort(-predictions)
product_index_sorted = np.argsort(-product_predictions)
brand_index_sorted = np.argsort(-brand_predictions)
label_list, label_name = get_result(index_sorted,
predictions_sorted, index2cid,
cid2name)
product_list, product_name = get_result(
product_index_sorted, product_predictions_sorted,
index2product, product2name)
brand_list, brand_name = get_result(brand_index_sorted,
brand_predictions_sorted,
index2brand, brand2name)
f.write(lines[i] + '\t' + ','.join(label_list) + '\t' +
','.join(label_name) + '\t' + ','.join(product_list) +
'\t' + ','.join(product_name) + '\t' +
','.join(brand_list) + '\t' + ','.join(brand_name) + '\n')
f.flush()