def train():
# load parameters
hparams = create_hparams()
start_time = time.time()
print("preparing train and dev data")
# load dict message
[
hparams.word2id, hparams.pos2id, hparams.role2id, hparams.id2word,
hparams.id2pos, hparams.id2role
] = data_helper.load_dict(hparams=hparams)
# [word, pos] ==> role
Train_word, Train_pos, Train_role, Dev_word, Dev_pos, Dev_role = data_helper.get_train(
hparams=hparams)
print("building model...")
with tf.Graph().as_default():
config = tf.ConfigProto(allow_soft_placement=True)
os.environ["CUDA_VISIBLE_DEVICES"] = str(hparams.gpu)
sess = tf.Session(config=config)
with sess.as_default():
with tf.device("/gpu:" + str(hparams.gpu)):
initializer = tf.random_uniform_initializer(-0.1, 0.1)
with tf.variable_scope("model",
reuse=None,
initializer=initializer):
model = BILSTM_CRF.bilstm_crf(hparams=hparams)
print("training model...")
sess.run(tf.global_variables_initializer())
model.train(sess, hparams, Train_word, Train_pos, Train_role,
Dev_word, Dev_pos, Dev_role)
print("final best f1 on valid dataset is: %f" % hparams.max_f1)
end_time = time.time()
print("time used %f (hour)" % ((end_time - start_time) / 3600))
return
def eval():
hparams = create_hparams()
# load dict message
[
hparams.word2id, hparams.pos2id, hparams.role2id, hparams.id2word,
hparams.id2pos, hparams.id2role
] = data_helper.load_dict(hparams=hparams)
print("Evaluation model...")
os.environ["CUDA_VISIBLE_DEVICES"] = str(hparams.gpu)
name = "1513764280"
checkpoint_dir = os.path.join('runs', name, "checkpoints")
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir)
graph = tf.Graph()
with graph.as_default():
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
with sess.as_default():
initializer = tf.random_uniform_initializer(-0.1, 0.1)
with tf.variable_scope("model",
reuse=None,
initializer=initializer):
model = BILSTM_CRF.bilstm_crf(hparams=hparams)
# model.saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
model.saver.restore(sess, ckpt.model_checkpoint_path)
sess.run(tf.tables_initializer())
print("Load ckpt %s file success!" % checkpoint_file)
type = 'test'
Test_word, Test_pos, Test_role = data_helper.get_test(
hparams, type)
model.eval(sess, hparams, Test_word, Test_pos, Test_role, type,
name)
return
def load_model(self):
'''
模型初始化，必须在构造方法中加载模型
'''
self.text2id, _ = load_dict(
os.path.join(DATA_PATH, 'MedicalClass/words_fr.dict'))
_, self.id2label = load_labeldict(
os.path.join(DATA_PATH, 'MedicalClass/label.dict'))
self.model = load_model(os.path.join(MODEL_PATH, 'model.h5'))
def load_model(self):
'''
模型初始化，必须在构造方法中加载模型
'''
self.sour2id, _ = load_dict(
os.path.join(DATA_PATH, 'MedicalQA/ask_fr.dict'))
self.targ2id, self.id2targ = load_dict(
os.path.join(DATA_PATH, 'MedicalQA/ans_fr.dict'))
self.session = tf.Session()
tf.saved_model.loader.load(self.session,
[tf.saved_model.tag_constants.SERVING],
os.path.join(MODEL_PATH, 'best'))
self.source_input = self.session.graph.get_tensor_by_name(
get_tensor_name('inputs'))
self.source_seq_len = self.session.graph.get_tensor_by_name(
get_tensor_name('source_sequence_length'))
self.target_seq_length = self.session.graph.get_tensor_by_name(
get_tensor_name('target_sequence_length'))
self.predictions = self.session.graph.get_tensor_by_name(
get_tensor_name('predictions'))
def deal_with_data(self):
'''
处理数据，没有可不写。
:return:
'''
# 加载数据
self.data = pd.read_csv(
os.path.join(DATA_PATH, 'MedicalClass/train.csv'))
# 划分训练集、测试集
self.train_data, self.valid_data = train_test_split(self.data,
test_size=0.01,
random_state=6,
shuffle=True)
self.text2id, _ = load_dict(
os.path.join(DATA_PATH, 'MedicalClass/words_fr.dict'))
# self.text2id, _ = load_dict(self.train_data)
self.label2id, _ = load_labeldict(
os.path.join(DATA_PATH, 'MedicalClass/label.dict'))
self.train_text, self.train_label = read_data(self.train_data,
self.text2id,
self.label2id)
self.val_text, self.val_label = read_data(self.valid_data,
self.text2id, self.label2id)
print('=*=数据处理完成=*=')
# RNN Size
# rnn_size = 64
rnn_size = 128
# Number of Layers
num_layers = 3
# Embedding Size
# encoding_embedding_size = 64
# decoding_embedding_size = 64
encoding_embedding_size = 128
decoding_embedding_size = 128
# Learning Rate
learning_rate = 0.001
# 超参数
que_dict, ans_dict = load_dict()
encoder_vocab_size = len(que_dict)
decoder_vocab_size = len(ans_dict)
class LSTM(object):
def __init__(self, batch_size):
self.inputs = tf.placeholder(tf.int32, [None, None], name='inputs')
self.targets = tf.placeholder(tf.int32, [None, None], name='targets')
self.source_sequence_length = tf.placeholder(
tf.int32, (None, ), name='source_sequence_length')
self.target_sequence_length = tf.placeholder(
tf.int32, (None, ), name='target_sequence_length')
self.learning_rate = tf.placeholder(tf.float32, name='learning_rate')
# self.max_target_sequence_length = tf.reduce_max(self.target_sequence_length, name='max_target_len')
self.max_target_sequence_length = tf.reduce_max(
# coding=utf-8
# author=yphacker
import tensorflow as tf
import config
from data_helper import load_dict
word_dict, word_dict_res = load_dict()
vocab_size = max(word_dict.values()) + 1
class CNN(object):
def __init__(self):
# 传值空间
self.input_x = tf.placeholder(tf.int32, shape=[None, None], name='input_x')
self.input_y = tf.placeholder(tf.int32, shape=[None], name='input_y')
self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
# define embedding layer
with tf.variable_scope('embedding'):
# 标准正态分布初始化
input_embedding = tf.Variable(
tf.truncated_normal(shape=[vocab_size, config.embedding_dim], stddev=0.1), name='encoder_embedding')
with tf.name_scope("cnn"):
# CNN layer
x_input_embedded = tf.nn.embedding_lookup(input_embedding, self.input_x)
conv = tf.layers.conv1d(x_input_embedded, config.num_filters, config.kernel_size, name='conv')
# global max pooling layer
pooling = tf.reduce_max(conv, reduction_indices=[1])
'Embedding dimensions of encoder and decoder inputs')
tf.app.flags.DEFINE_float('learning_rate', 0.001, 'Learning rate')
tf.app.flags.DEFINE_float('learning_rate_decay_factor', 0.99,
'Learning rate decay factor rate')
tf.app.flags.DEFINE_integer('batch_size', 128, 'Batch size')
tf.app.flags.DEFINE_integer('numEpochs', 25, 'Maximum # of training epochs')
tf.app.flags.DEFINE_integer('steps_per_checkpoint', 100,
'Save model checkpoint every this iteration')
tf.app.flags.DEFINE_string('model_dir', 'seq2seq_model/',
'Path to save model checkpoints')
tf.app.flags.DEFINE_string('model_name', 'seq2seq.ckpt',
'File name used for model checkpoints')
FLAGS = tf.app.flags.FLAGS
word2idx, idx2word = load_dict()
# data_path = '../data/train_data_idx.pkl'
data_path = '../data/train_new_data_idx.pkl'
trainingSamples = loadDataset(data_path)
# test_path = '../data/test_data_idx.pkl'
test_path = '../data/test_new_data_idx26.pkl'
testingSamples = loadDataset(test_path)
model = Seq2SeqModel(FLAGS.rnn_size,
FLAGS.num_layers,
FLAGS.embedding_size,
FLAGS.learning_rate,
FLAGS.learning_rate_decay_factor,
word2idx,
mode='train',
def __init__(self):
super(Processor, self).__init__()
self.word_dict, self.word_dict_re = load_dict()