test_data_list_path=conf_dict['test_data_path'])
predict_infer(conf_dict, data_generator, predict_data_path, \
predict_result_path, model_path)
if __name__ == '__main__':
#
Load configuration file
#加载配置文件
parser = argparse.ArgumentParser()#打印
parser.add_argument("--conf_path", type=str,
help="conf_file_path_for_model. (default: %(default)s)",
required=True)
parser.add_argument("--model_path", type=str,
help="model_path", required=True)
parser.add_argument("--predict_file", type=str,
help="the_file_to_be_predicted", required=True)
parser.add_argument("--result_file", type=str,
default='', help="the_file_of_predicted_results")
args = parser.parse_args()
conf_dict = conf_lib.load_conf(args.conf_path)#加载运行（？）路径
model_path = args.model_path
predict_data_path = args.predict_file
predict_result_path = args.result_file
for input_path in [model_path, predict_data_path]:
if not os.path.exists(input_path):
raise ValueError("%s not found." % (input_path))
main(conf_dict, model_path, predict_data_path, predict_result_path)#运行主函数
def save_tf_model(sess, epoch):
saver = tf.train.Saver()
saver.save(sess, "model/spo_model/model.ckpt", global_step=epoch)
def main(conf_dict, use_cuda=False):
"""Train main function"""
data_generator = spo_data_reader.DataReader(
wordemb_dict_path=conf_dict['word_idx_path'],
postag_dict_path=conf_dict['postag_dict_path'],
label_dict_path=conf_dict['so_label_dict_path'],
p_eng_dict_path=conf_dict['label_dict_path'],
train_data_list_path=conf_dict['spo_train_data_path'],
test_data_list_path=conf_dict['spo_test_data_path'])
data_generator.load_train_data()
train(conf_dict, data_generator, use_cuda=use_cuda)
if __name__ == '__main__':
# Load the configuration file
parser = argparse.ArgumentParser()
parser.add_argument("--conf_path", type=str,
help="conf_file_path_for_model. (default: %(default)s)",
required=True)
args = parser.parse_args()
conf_dict = conf_lib.load_conf(args.conf_path)
use_gpu = True if conf_dict.get('use_gpu', 'False') == 'True' else False
main(conf_dict, use_cuda=use_gpu)
def train(conf_dict, data_reader, use_cuda=False):
"""
Training of so labeling model
"""
# input data layer
word = fluid.layers.data(
name='word_data', shape=[1], dtype='int64', lod_level=1)
postag = fluid.layers.data(
name='token_pos', shape=[1], dtype='int64', lod_level=1)
p_word = fluid.layers.data(
name='p_word', shape=[1], dtype='int64', lod_level=1)
# label
target = fluid.layers.data(
name='target', shape=[1], dtype='int64', lod_level=1)
# embedding + lstm
feature_out = spo_model.db_lstm(data_reader, word, \
postag, p_word, conf_dict)
# loss function
# crf layer
mix_hidden_lr = float(conf_dict['mix_hidden_lr'])
crf_cost = fluid.layers.linear_chain_crf(
input=feature_out,
label=target,
param_attr=fluid.ParamAttr(name='crfw', learning_rate=mix_hidden_lr))
avg_cost = fluid.layers.mean(crf_cost)
# optimizer
sgd_optimizer = fluid.optimizer.AdamOptimizer(
learning_rate=2e-3, )
sgd_optimizer.minimize(avg_cost)
crf_decode = fluid.layers.crf_decoding(
input=feature_out, param_attr=fluid.ParamAttr(name='crfw'))
train_batch_reader = paddle.batch(
paddle.reader.shuffle(data_reader.get_train_reader(), buf_size=8192),
batch_size=conf_dict['batch_size'])
place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
feeder = fluid.DataFeeder(feed_list=[word, postag, p_word, target], place=place)
exe = fluid.Executor(place)
save_dirname = conf_dict['spo_model_save_dir']
def train_loop(main_program, trainer_id=0):
"""start train loop"""
exe.run(fluid.default_startup_program())
start_time = time.time()
batch_id = 0
for pass_id in six.moves.xrange(conf_dict['pass_num']):
pass_start_time = time.time()
cost_sum, cost_counter = 0, 0
for data in train_batch_reader():
cost = exe.run(main_program, feed=feeder.feed(data), fetch_list=[avg_cost])
cost = cost[0]
cost_sum += cost
cost_counter += 1
if batch_id % 10 == 0 and batch_id != 0:
#sys.stderr.write("batch %d finished, second per batch: %02f\n" % (
# batch_id, (time.time() - start_time) / batch_id))
# cost expected, training over
if float(cost) < 1:
save_path = os.path.join(save_dirname, 'final')
fluid.io.save_inference_model(save_path, ['word_data', 'token_dist', 'p_word'],
[feature_out], exe, params_filename='params')
return
batch_id = batch_id + 1
# save the model once each pass ends
pass_avg_cost = cost_sum / cost_counter if cost_counter > 0 else 0.0
#sys.stderr.write("%d pass end, cost time: %02f, avg_cost: %f" % (
# pass_id, time.time() - pass_start_time, pass_avg_cost))
save_path = os.path.join(save_dirname, 'pass_%04d-%f' %
(pass_id, pass_avg_cost))
fluid.io.save_inference_model(save_path, ['word_data', 'token_pos', 'p_word'],
[feature_out], exe, params_filename='params')
else:
# pass times complete and the training is over
save_path = os.path.join(save_dirname, 'final')
fluid.io.save_inference_model(save_path, ['word_data', 'token_pos', 'p_word'],
[feature_out], exe, params_filename='params')
return
train_loop(fluid.default_main_program())
def main(conf_dict, use_cuda=False):
"""Train main function"""
if use_cuda and not fluid.core.is_compiled_with_cuda():
return
data_generator = spo_data_reader.DataReader(
wordemb_dict_path=conf_dict['word_idx_path'],
postag_dict_path=conf_dict['postag_dict_path'],
label_dict_path=conf_dict['so_label_dict_path'],
p_eng_dict_path=conf_dict['label_dict_path'],
train_data_list_path=conf_dict['spo_train_data_path'],
test_data_list_path=conf_dict['spo_test_data_path'])
train(conf_dict, data_generator, use_cuda=use_cuda)
if __name__ == '__main__':
# Load the configuration file
parser = argparse.ArgumentParser()
parser.add_argument("--conf_path", type=str,
help="conf_file_path_for_model. (default: %(default)s)",
required=True)
args = parser.parse_args()
conf_dict = conf_lib.load_conf(args.conf_path)
use_gpu = True if conf_dict.get('use_gpu', 'False') == 'True' else False
main(conf_dict, use_cuda=use_gpu)