def main():
config = Config()
print('Prepare data for train and dev ... ')
train_dev_split(config.original_file, config.train_file, config.dev_file,
config.vocab_file, config.split_ratio)
print('Prepare data sucessfully!')
lstm_config = Config()
blstm_model = LSTM_Dynamic(lstm_config)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
if lstm_config.if_test:
print('Start load dev data ...')
(X_test, y_test) = load_data(lstm_config.dev_file)
print('Loading sucess!')
if len(X_test) < lstm_config.batch_size:
for i in range(0, lstm_config.batch_size - len(X_test)):
X_test.append([0])
y_test.append([0])
seq_len_test = list(map(lambda x: len(x), X_test))
print('Target to special model to test')
test_model = os.path.join(lstm_config.model_path, "models_epoch38")
print('Start do predicting...')
blstm_model.test(sess, test_model, X_test, y_test, seq_len_test,
lstm_config.vocab_file,
lstm_config.model_path + 'result/')
def inference_main(contents):
'''
:param contents: 一维列表，每行是一个字符串
:return:
'''
config = Config()
config.batch_size = 10000
config.if_inference = True
print('Prepare data for train and dev ... ')
train_dev_split(config.original_file, config.train_file, config.dev_file,
config.vocab_file, config.split_ratio)
print('Prepare data sucessfully!')
blstm_model = LSTM_Dynamic(config)
gpu_options =tf.GPUOptions(per_process_gpu_memory_fraction=0.5, allow_growth=True)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=True)) as sess:
if config.if_inference:
x_infer = prepare_modelinputs(contents,y_flag=False)
if len(x_infer) < config.batch_size:
for i in range(0, config.batch_size - len(x_infer)):
x_infer.append([0])
print('Target to special model to test')
test_model = os.path.join(config.model_path, "models_epoch39")
print('Start do predicting...','\n'*10)
saver = tf.train.Saver(tf.trainable_variables())
saver.restore(sess, test_model)
y_pred = infer(blstm_model, sess, x_infer, list(map(len, x_infer)))
return y_pred
def main():
config = Config()
print('Prepare data for train and dev ... ')
train_dev_split(config.original_file, config.train_file, config.dev_file,
config.vocab_file, config.split_ratio)
print('Prepare data sucessfully!')
lstm_config = Config()
rnn_model = LSTM_Dynamic(lstm_config)
gpu_options = tf.GPUOptions(allow_growth=True)
gpu_options =tf.GPUOptions(per_process_gpu_memory_fraction=0.5, allow_growth=True) ##每个gpu占用0.8 的显存
config=tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=True)
with tf.Session(config=config) as sess:
if lstm_config.if_train:
init=tf.global_variables_initializer()
sess.run(init)
(X_train, y_train) = load_data(lstm_config.train_file)
if len(X_train) < lstm_config.batch_size:
for i in range(0, lstm_config.batch_size - len(X_train)):
X_train.append([0])
y_train.append([0])
seq_len_train = list(map(lambda x: len(x), X_train))
rnn_model.train_epoch(sess, lstm_config.train_file, X_train,
y_train, seq_len_train,
lstm_config.model_path)
print('Success for preparing data')
def main(test_model):
config = Config()
config.batch_size = 1024
config.hidden_size = 64
config.vocab_size = 26
config.embed_size = 320
config.max_epochs = 100
config.label_kinds = 2
config.if_train = True
config.if_test = True
config.is_biLSTM = True
config.max_seqlen = 20
config.original_file = '../data/most_frequent_words_label.txt'
config.train_file = '../data/most_frequent_words_label_train.txt'
config.dev_file = '../data/most_frequent_words_label_dev'
config.vocab_file = '../data/vocab.txt'
config.model_path = 'models/Transformer_softmax_lstm/'
config.split_ratio = 0.8
print('Prepare data for train and dev ... ')
train_dev_split(config.original_file, config.train_file, config.dev_file,
config.vocab_file, config.split_ratio)
print('Prepare data sucessfully!')
model = Transformer(config)
gpu_options = tf.GPUOptions(allow_growth=True)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=0.5, allow_growth=True
) ##每个gpu占用0.8 的显存
tf_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True)
with tf.Session(config=tf_config) as sess:
if config.if_test:
init = tf.global_variables_initializer()
sess.run(init)
(X_test, y_test) = load_data(config.dev_file)
if len(X_test) < config.batch_size:
for i in range(0, config.batch_size - len(X_test)):
X_test.append([0])
y_test.append([0])
seq_len_test = list(map(lambda x: len(x), X_test))
print('Target to special model to test')
print('Start do predicting...')
model.test(sess, test_model, X_test, y_test, seq_len_test,
config.vocab_file, config.model_path + 'result/')
print('Success for preparing data')
def main():
config = Config()
config.if_train = True
print('Prepare data for train and dev ... ')
train_dev_split(config.original_file, config.train_file, config.dev_file,
config.vocab_file, config.split_ratio)
print('Prepare data sucessfully!')
rnn_model = LSTM_Dynamic(config)
gpu_options = tf.GPUOptions(allow_growth=True)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5, allow_growth=True) ##每个gpu占用0.8 的显存
tf_config=tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=True)
with tf.Session(config=tf_config) as sess:
if config.if_train:
model_input_path = 'model_input' + str(config.embed_size) + os.path.basename(config.train_file)
if not os.path.exists(model_input_path):
prepare_and_write_modelinputs(config.train_file, model_input_path)
train_epoch(rnn_model, sess, model_input_path, config.model_path)
