def train(params):
# GPU资源配置
config_gpu()
# 读取vocab训练
print("Building the model ...")
vocab = Vocab(params["vocab_path"], params["vocab_size"])
# 构建模型
print("Building the model ...")
# model = Seq2Seq(params)
model = PGN(params)
print("Creating the batcher ...")
# dataset = batcher(params["train_seg_x_dir"], params["train_seg_y_dir"], vocab, params)
# print('dataset is ', dataset)
# 获取保存管理者
print("Creating the checkpoint manager")
checkpoint = tf.train.Checkpoint(Seq2Seq=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint,
checkpoint_dir,
max_to_keep=5)
checkpoint.restore(checkpoint_manager.latest_checkpoint)
if checkpoint_manager.latest_checkpoint:
print("Restored from {}".format(checkpoint_manager.latest_checkpoint))
else:
print("Initializing from scratch.")
# 训练模型
print("Starting the training ...")
train_model(model, vocab, params, checkpoint_manager)
def decoder():
# 配置GPU
use_gpu = True
config_gpu(use_gpu=use_gpu)
# 读取字典 和 词向量矩阵
vocab = Vocab(config.path_vocab, config.vocab_size)
wvtool = WVTool(ndim=config.emb_dim)
embedding_matrix = wvtool.load_embedding_matrix(path_embedding_matrix=config.path_embedding_matrixt)
# 构建模型
logger.info('构建Seq2Seq模型 ...')
model=Seq2Seq(config.beam_size,embedding_matrix=embedding_matrix)
# 存档点管理
ckpt = tf.train.Checkpoint(Seq2Seq=model)
ckpt_manager = tf.train.CheckpointManager(checkpoint=ckpt, directory=config.dir_ckpt, max_to_keep=10)
if ckpt_manager.latest_checkpoint:
ckpt.restore(ckpt_manager.latest_checkpoint)
logger.info('decoder模型存档点加载自: {}'.format(ckpt_manager.latest_checkpoint))
else:
logger.info('无可加载的存档点')
# 获取训练数据
batcher = Batcher(config.path_seg_test, vocab, mode='decode',
batch_size=config.beam_size, single_pass=True)
time.sleep(20)
# 训练模型
# 输入：训练数据barcher，模型，词表，存档点，词向量矩阵
batch_decode(batcher, model=model,vocab=vocab)
def train(params):
# GPU资源配置
config_gpu(use_cpu=False, gpu_memory=params['gpu_memory'])
# 读取vocab训练
print("Building the model ...")
vocab = Vocab(params["vocab_path"], params["max_vocab_size"])
params['vocab_size'] = vocab.count
# 构建模型
print("Building the model ...")
# model = Seq2Seq(params)
model = PGN(params)
print("Creating the batcher ...")
dataset = batcher(vocab, params)
# print('dataset is ', dataset)
# 获取保存管理者
print("Creating the checkpoint manager")
checkpoint = tf.train.Checkpoint(PGN=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint,
params['checkpoint_dir'],
max_to_keep=5)
checkpoint.restore(checkpoint_manager.latest_checkpoint)
if checkpoint_manager.latest_checkpoint:
print("Restored from {}".format(checkpoint_manager.latest_checkpoint))
else:
print("Initializing from scratch.")
# 训练模型
print("Starting the training ...")
train_model(model, dataset, params, checkpoint_manager)
def test(params):
assert params["mode"].lower() in [
"test", "eval"
], "change training mode to 'test' or 'eval'"
if params['decode_mode'] == 'beam':
assert params["beam_size"] == params[
"batch_size"], "Beam size must be equal to batch_size, change the params"
# GPU资源配置
config_gpu(use_cpu=True)
print("Building the model ...")
model = PGN(params)
print("Creating the vocab ...")
vocab = Vocab(params["vocab_path"], params["vocab_size"])
params['vocab_size'] = vocab.count
print("Creating the checkpoint manager")
checkpoint = tf.train.Checkpoint(PGN=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint,
checkpoint_dir,
max_to_keep=5)
checkpoint.restore(checkpoint_manager.latest_checkpoint)
if checkpoint_manager.latest_checkpoint:
print("Restored from {}".format(checkpoint_manager.latest_checkpoint))
else:
print("Initializing from scratch.")
print("Model restored")
results = predict_result(model, params, vocab, params['result_save_path'])
print('save result to :{}'.format(params['result_save_path']))
print('save result :{}'.format(results[:5]))
def test(params):
assert params["mode"].lower() in ["test", "eval"], "change training mode to 'test' or 'eval'"
assert params["beam_size"] == params["batch_size"], "Beam size must be equal to batch_size, change the params"
# GPU资源配置
config_gpu(use_cpu=True)
print("Building the model ...")
model = Seq2Seq(params)
print("Creating the vocab ...")
vocab = Vocab(params["vocab_path"], params["vocab_size"])
params['vocab_size'] = vocab.count
print("Creating the checkpoint manager")
checkpoint = tf.train.Checkpoint(Seq2Seq=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint, checkpoint_dir, max_to_keep=5)
checkpoint.restore(checkpoint_manager.latest_checkpoint)
if checkpoint_manager.latest_checkpoint:
print("Restored from {}".format(checkpoint_manager.latest_checkpoint))
else:
print("Initializing from scratch.")
print("Model restored")
if params['greedy_decode']:
predict_result(model, params, vocab, params['result_save_path'])
else:
b = bream_test_batch_generator(params["beam_size"])
results = []
for batch in b:
best_hyp = beam_decode(model, batch, vocab, params)
results.append(best_hyp.abstract)
save_predict_result(results, params['result_save_path'])
print('save result to :{}'.format(params['result_save_path']))
def test(params):
assert params["mode"].lower() in ["test", "eval"], "change training mode to 'test' or 'eval'"
assert params["beam_size"] == params["batch_size"], "Beam size must be equal to batch_size, change the params"
# GPU资源配置
config_gpu(use_cpu=True)
print("Building the model ...")
model = PGN(params)
print("Creating the vocab ...")
vocab = Vocab(params["vocab_path"], params["vocab_size"])
params['vocab_size'] = vocab.count
print("Creating the batcher ...")
b = batcher(vocab, params)
print("Creating the checkpoint manager")
checkpoint = tf.train.Checkpoint(PGN=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint, checkpoint_dir, max_to_keep=5)
checkpoint.restore(checkpoint_manager.latest_checkpoint)
if checkpoint_manager.latest_checkpoint:
print("Restored from {}".format(checkpoint_manager.latest_checkpoint))
else:
print("Initializing from scratch.")
print("Model restored")
if params['mode'] == 'eval' or params['mode'] == 'test':
for batch in b:
#print("batch is:\n", batch)
yield beam_decode(model, batch, vocab, params)
else:
for batch in b:
print(beam_decode(model, batch, vocab, params))
def train(params):
config_gpu()
vocab_to_index, _ = load_vocab(word2vec_model_path)
params['vocab_size'] = len(vocab_to_index)
seq2seq = Seq2Seq(params)
checkpoint = tf.train.Checkpoint(model=seq2seq)
checkpoint_manager = tf.train.CheckpointManager(checkpoint,
checkpoint_dir,
max_to_keep=5)
train_model(seq2seq, params, vocab_to_index, checkpoint_manager)
def train(params):
config_gpu()
vocab, _ = get_vocab(model_path_trained)
params['vocab_size'] = len(vocab)
print("Building the model")
model = Seq2Seq(params)
checkpoint = tf.train.Checkpoint(Seq2Seq=model)
check_point_manager = tf.train.CheckpointManager(checkpoint,
checkpoint_dir,
max_to_keep=5)
train_model(model, vocab, params, check_point_manager)
def train(params):
# GPU资源配置
config_gpu()
# 读取vocab训练
vocab = Vocab(params["vocab_path"], params["vocab_size"])
params['vocab_size'] = vocab.count
# 构建模型
print("Building the model ...")
model = Seq2Seq(params)
# 获取保存管理者
checkpoint = tf.train.Checkpoint(Seq2Seq=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint, params['checkpoint_dir'], max_to_keep=5)
# 训练模型
train_model(model, vocab, params, checkpoint_manager)
def train(params):
# GPU资源配置
config_gpu()
# 读取vocab训练
print("Building the model ...")
vocab = Vocab(params["vocab_path"], params["max_vocab_size"])
params['vocab_size'] = vocab.count
# 构建模型
print("Building the model ...")
# model = Seq2Seq(params)
model = PGN(params)
print("Creating the batcher ...")
dataset, params['steps_per_epoch'] = batcher(vocab, params)
# print('dataset is ', dataset)
# 获取保存管理者
print("Creating the checkpoint manager")
checkpoint = tf.train.Checkpoint(PGN=model)
checkpoint_manager = tf.train.CheckpointManager(checkpoint,
params['checkpoint_dir'],
max_to_keep=5)
checkpoint.restore(checkpoint_manager.latest_checkpoint)
if checkpoint_manager.latest_checkpoint:
print("Restored from {}".format(checkpoint_manager.latest_checkpoint))
params["trained_epoch"] = int(checkpoint_manager.latest_checkpoint[-1])
else:
print("Initializing from scratch.")
params["trained_epoch"] = 1
# 学习率衰减
params["learning_rate"] *= np.power(0.95, params["trained_epoch"])
print('learning_rate:{}'.format(params["learning_rate"]))
# 训练模型
print("Starting the training ...")
train_model(model, dataset, params, checkpoint_manager)
# passing the concatenated vector to the GRU
output, state = self.gru(x)
# output shape == (batch_size * 1, hidden_size)
output = tf.reshape(output, (-1, output.shape[2]))
# output shape == (batch_size, vocab)
prediction = self.fc(output)
return prediction, state
if __name__ == '__main__':
# GPU资源配置
config_gpu()
# 获得参数
params = get_params()
# 读取vocab训练
vocab = Vocab(params["vocab_path"], params["vocab_size"])
# 计算vocab size
vocab_size = vocab.count
# 使用GenSim训练好的embedding matrix
embedding_matrix = load_word2vec_file(save_wv_model_path)
input_sequence_len = 250
BATCH_SIZE = 64
embedding_dim = 500
units = 1024
# 编码器结构
