def test_bleu(count):
"""测试bleu, 这个方法我们不看"""
print("bleu test mode")
from nltk.translate.bleu_score import sentence_bleu
from tqdm import tqdm
# 准备数据
print('准备数据')
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有数据 {} 条'.format(total_size))
# bleu设置0的话，默认对所有样本采样
if count <= 0:
count = total_size
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
]
with tf.Session() as sess:
# 构建模型
model = create_model(sess, True)
model.batch_size = 1
# 初始化变量
sess.run(tf.initialize_all_variables())
model.saver.restore(sess,
os.path.join(FLAGS.model_dir, FLAGS.model_name))
total_score = 0.0
for i in tqdm(range(count)):
# 选择一个要训练的bucket
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if buckets_scale[i] > random_number
])
data, _ = model.get_batch_data(bucket_dbs, bucket_id)
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_id, data)
_, _, output_logits = model.step(sess, encoder_inputs,
decoder_inputs, decoder_weights,
bucket_id, True)
outputs = [
int(np.argmax(logit, axis=1)) for logit in output_logits
]
ask, _ = data[0]
all_answers = bucket_dbs[bucket_id].all_answers(ask)
ret = data_utils.indice_sentence(outputs)
if not ret:
continue
references = [list(x) for x in all_answers]
score = sentence_bleu(references, list(ret), weights=(1.0, ))
total_score += score
print('BLUE: {:.2f} in {} samples'.format(total_score / count * 10,
count))
def test_bleu(count):
u'测试bleu'
from nltk.translate.bleu_score import sentence_bleu
from tqdm import tqdm
print(u'准备数据')
bucket_dbs = data_utils.read_bucket_dbs(
FLAGS.buckets_dir) #FLAGS.buckets_dir
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print(u'bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print(u'共有数据 {} 条'.format(total_size))
if (count <= 0):
count = total_size
buckets_scale = [(sum(bucket_sizes[:(i + 1)]) / total_size)
for i in range(len(bucket_sizes))]
with tf.Session() as sess:
model = create_model(sess, True)
model.batch_size = 1
sess.run(tf.initialize_all_variables())
model.saver.restore(sess,
os.path.join(FLAGS.model_dir, FLAGS.model_name))
total_score = 0.0
for i in tqdm(range(count)):
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if (buckets_scale[i] > random_number)
])
(data, _) = model.get_batch_data(bucket_dbs, bucket_id)
(encoder_inputs, decoder_inputs,
decoder_weights) = model.get_batch(bucket_dbs, bucket_id, data)
(_, _, output_logits) = model.step(sess, encoder_inputs,
decoder_inputs, decoder_weights,
bucket_id, True)
outputs = [
int(np.argmax(logit, axis=1)) for logit in output_logits
]
(ask, _) = data[0]
all_answers = bucket_dbs[bucket_id].all_answers(ask)
ret = data_utils.indice_sentence(outputs)
if (not ret):
continue
references = [list(x) for x in all_answers]
score = sentence_bleu(references, list(ret), weights=(1.0, ))
total_score += score
print(u'BLUE: {:.2f} in {} samples'.format(
((total_score / count) * 10), count))
def train():
"""训练模型"""
# 准备数据
print("train mode")
print('准备数据')
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有数据 {} 条'.format(total_size))
# 开始建模与训练
with tf.Session(config=tf.ConfigProto(allow_soft_placement=False,
gpu_options=gpu_options)) as sess:
# 构建模型
model = create_model(sess, False)
# 初始化变量
sess.run(tf.initialize_all_variables())
ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir)
#print("ckpt path : ", ckpt.model_checkpoint_path)
if ckpt != None:
print("读取模型 : ", ckpt.model_checkpoint_path)
model.saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("not exist old model")
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
]
# 开始训练
metrics = ' '.join(
['\r[{}]', '{:.1f}%', '{}/{}', 'loss={:.3f}', '{}/{}'])
bars_max = 20
with tf.device('/gpu:0'):
for epoch_index in range(1, FLAGS.num_epoch + 1):
print('Epoch {}:'.format(epoch_index))
time_start = time.time()
epoch_trained = 0
batch_loss = []
while True:
# 选择一个要训练的bucket
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if buckets_scale[i] > random_number
])
data, data_in = model.get_batch_data(bucket_dbs, bucket_id)
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_dbs, bucket_id, data)
_, step_loss, output = model.step(sess, encoder_inputs,
decoder_inputs,
decoder_weights,
bucket_id, False)
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss)
time_now = time.time()
time_spend = time_now - time_start
time_estimate = time_spend / (epoch_trained /
FLAGS.num_per_epoch)
percent = min(100,
epoch_trained / FLAGS.num_per_epoch) * 100
bars = math.floor(percent / 100 * bars_max)
sys.stdout.write(
metrics.format('=' * bars + '-' * (bars_max - bars),
percent,
epoch_trained, FLAGS.num_per_epoch,
np.mean(batch_loss),
data_utils.time(time_spend),
data_utils.time(time_estimate)))
sys.stdout.flush()
if epoch_trained >= FLAGS.num_per_epoch:
model.saver.save(sess,
os.path.join(FLAGS.model_dir,
FLAGS.model_name),
global_step=epoch_index)
break
print('\n')
def train():
u'训练模型'
print(u'准备数据')
bucket_dbs = data_utils.read_bucket_dbs(
FLAGS.buckets_dir) #FLAGS.buckets_dir
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print(u'bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print(u'共有数据 {} 条'.format(total_size))
with tf.Session() as sess:
model = create_model(sess, False)
sess.run(tf.initialize_all_variables())
buckets_scale = [(sum(bucket_sizes[:(i + 1)]) / total_size)
for i in range(len(bucket_sizes))]
metrics = u' '.join(
[u'\r[{}]', u'{:.1f}%', u'{}/{}', u'loss={:.3f}', u'{}/{}'])
bars_max = 20
for epoch_index in range(1, (FLAGS.num_epoch + 1)):
print(u'Epoch {}:'.format(epoch_index))
time_start = time.time()
epoch_trained = 0
batch_loss = []
while True:
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if (buckets_scale[i] > random_number)
])
(data, data_in) = model.get_batch_data(bucket_dbs, bucket_id)
(encoder_inputs, decoder_inputs,
decoder_weights) = model.get_batch(bucket_dbs, bucket_id,
data)
(_, step_loss,
output) = model.step(sess, encoder_inputs, decoder_inputs,
decoder_weights, bucket_id, False)
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss)
time_now = time.time()
time_spend = (time_now - time_start)
time_estimate = (time_spend /
(epoch_trained / FLAGS.num_per_epoch))
percent = (min(100,
(epoch_trained / FLAGS.num_per_epoch)) * 100)
bars = math.floor(((percent / 100) * bars_max))
#sys.stdout.write(metrics.format(((u'=' * bars) + (u'-' * (bars_max - bars))), float(percent), epoch_trained,
# FLAGS.num_per_epoch, np.mean(batch_loss), data_utils.time(time_spend), data_utils.time(time_estimate)))
sys.stdout.flush()
if (epoch_trained >= FLAGS.num_per_epoch):
break
print(u'\n')
if (not os.path.exists(FLAGS.model_dir)):
os.makedirs(FLAGS.model_dir)
parser = argparse.ArgumentParser()
parser.add_argument('--model_dir',
type=str,
default='',
help='ouput data puth')
ARGS = None
ARGS, _ = parser.parse_known_args()
model_path = ARGS.model_dir
save_path = os.path.join(model_path, "model1")
model.saver.save(sess, save_path)
def train():
# 流程
# 1.数据预处理
# 2.seq2seq
# ========================================================
# 准备数据
print("train mode.......")
print('准备数据')
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
# 数据预处理
# buckets_dir 训练数据目录
# 获取列表，列表里是四个桶，每个桶里有各自的数据库内容
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
# 每个桶的数据量添加
bucket_sizes = []
for i in range(len(buckets)):
# 语句的尺寸
bucket_size = bucket_dbs[i].size # 不同的桶的数据量
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
# 所有样本的数目
total_size = sum(bucket_sizes) # 获取所有的样本数目
print('共有数据 {} 条'.format(total_size))
# 开始建模与训练
gpu_options = tf.GPUOptions(
allow_growth=True, # 允许GPU分配是一种增量分配的方式
per_process_gpu_memory_fraction=calc_gpu_fraction(FLAGS.gpu_fraction))
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)) as sess:
# 构建模型（每个桶对应一个训练对象\损失函数\summary相关信息；但是这四套代码是参数共享的）
model = create_model(sess, False)
# 初始化变量&模型恢复
print("开始进行模型初始化以及模型恢复操作.....")
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Load old model from : ", ckpt.model_checkpoint_path)
model.saver.restore(sess, ckpt.model_checkpoint_path)
model.saver.recover_last_checkpoints(
ckpt.all_model_checkpoint_paths)
else:
print("Not exist old model")
# 计算每个桶的样本的累计占比(1号桶的占比， 1+2号桶的占比， 1+2+3号桶的占比， 1+2+3+4号桶的占比)
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
]
# 开始训练
metrics = ' '.join(
['\r[{}]', '{:.1f}%', '{}/{}', 'loss={:.3f}', '{}/{}'])
# 设置bars_max跟踪进度
bars_max = 20
writer = tf.summary.FileWriter('log', graph=sess.graph)
merges = []
# 针对每个桶(每个训练对象)获取对应的summary的的可视化输出对象
for b_idx in model.bucket_to_summary_list:
merges.append(tf.summary.merge(
model.bucket_to_summary_list[b_idx]))
print("开始模型训练.....")
with tf.device('/gpu:0'):
for epoch_index in range(1, FLAGS.num_epoch + 1):
print('Epoch {}:'.format(epoch_index))
# 获取开始的时间
time_start = time.time()
# 设置开始的进度为 0
epoch_trained = 0
batch_loss = []
while True:
# 随机选择一个要训练的bucket
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if buckets_scale[i] > random_number
])
# 获取数据（从随机的桶中获取数据，获取batch_size: 16条数据）
data, _ = model.get_batch_data(bucket_dbs, bucket_id)
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_id, data)
# run 迭代训练
_, step_loss, summary_merge, output = model.step(
sess, encoder_inputs, decoder_inputs, decoder_weights,
bucket_id, False, merges[bucket_id])
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss)
# 获取现在的时间
time_now = time.time()
# 获取经历的时间
time_spend = time_now - time_start
# 获取时间的进度
time_estimate = time_spend / (epoch_trained /
FLAGS.num_per_epoch)
# 获取现在的进度比例
percent = min(100,
epoch_trained / FLAGS.num_per_epoch) * 100
# bars该显示多少个计算，最多显示20个
bars = math.floor(percent / 100 * bars_max)
# 进行输出操作，显示=号，-号的数量不同
sys.stdout.write(
metrics.format(
'=' * int(bars) + '-' * int(bars_max - bars),
percent, epoch_trained, FLAGS.num_per_epoch,
np.mean(batch_loss), data_utils.time(time_spend),
data_utils.time(time_estimate)))
# 进行输出
sys.stdout.flush()
if summary_merge is not None:
writer.add_summary(summary_merge,
global_step=epoch_index)
if epoch_trained >= FLAGS.num_per_epoch:
model.saver.save(sess,
os.path.join(FLAGS.model_dir,
FLAGS.model_name),
global_step=epoch_index)
break
print('\n')
# 最终再来一次模型持久化输出
model.saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_name))
def train():
"""训练模型"""
# 准备数据
print('准备数据')
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有数据 {} 条'.format(total_size))
# 开始建模与训练
with tf.Session() as sess:
# 构建模型
model = create_model(sess, False)
# 初始化变量
sess.run(tf.initialize_all_variables())
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
]
# 开始训练
metrics = ' '.join([
'\r[{}]',
'{:.1f}%',
'{}/{}',
'loss={:.3f}',
'{}/{}',
'learning rate={:.5f}'
])
bars_max = 20
for epoch_index in range(1, FLAGS.num_epoch + 1):
print('Epoch {}:'.format(epoch_index))
time_start = time.time()
epoch_trained = 0
batch_loss = []
#
previous_losses=[]
current_step=0
loss=0
while True:
# 选择一个要训练的bucket
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if buckets_scale[i] > random_number
])
data, data_in = model.get_batch_data(
bucket_dbs,
bucket_id
)
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_dbs,
bucket_id,
data
)
_, step_loss, output = model.step(
sess,
encoder_inputs,
decoder_inputs,
decoder_weights,
bucket_id,
False
)
loss=step_loss/FLAGS.steps_per_checkpoint
current_step+=1
if current_step % FLAGS.steps_per_checkpoint == 0:
if len(previous_losses)>2 and loss>max(previous_losses[-3:]):
sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
loss=0
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
model.saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_name))
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss)
time_now = time.time()
time_spend = time_now - time_start
time_estimate = time_spend / (epoch_trained / FLAGS.num_per_epoch)
percent = min(100, epoch_trained / FLAGS.num_per_epoch) * 100
bars = math.floor(percent / 100 * bars_max)
sys.stdout.write(metrics.format(
'=' * bars + '-' * (bars_max - bars),
percent,
epoch_trained, FLAGS.num_per_epoch,
np.mean(batch_loss),
data_utils.time(time_spend), data_utils.time(time_estimate),model.learning_rate.eval()
))
sys.stdout.flush()
if epoch_trained >= FLAGS.num_per_epoch:
break
print('\n')
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
model.saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_name))
def train():
"""训练模型"""
# 准备数据
print('准备数据')
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有数据 {} 条'.format(total_size))
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.66)
config = tf.ConfigProto(gpu_options=gpu_options)
# 防止 out of memory
config.gpu_options.allocator_type = 'BFC'
# 开始建模与训练
with tf.Session() as sess:
# 构建模型
model = create_model(sess, False)
# 初始化变量
sess.run(tf.initialize_all_variables())
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
]
# 开始训练
metrics = ' '.join([
'\r[{}]',
'{:.1f}%',
'{}/{}',
'loss={:.3f}',
'{}/{}'
])
bars_max = 20
for epoch_index in range(1, FLAGS.num_epoch + 1):
print('Epoch {}:'.format(epoch_index))
time_start = time.time()
epoch_trained = 0
batch_loss = []
while True:
# 选择一个要训练的bucket
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if buckets_scale[i] > random_number
])
data, data_in = model.get_batch_data(
bucket_dbs,
bucket_id
)
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_dbs,
bucket_id,
data
)
_, step_loss, output = model.step(
sess,
encoder_inputs,
decoder_inputs,
decoder_weights,
bucket_id,
False
)
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss)
time_now = time.time()
time_spend = time_now - time_start
time_estimate = time_spend / (epoch_trained / FLAGS.num_per_epoch)
percent = min(100, epoch_trained / FLAGS.num_per_epoch) * 100
bars = math.floor(percent / 100 * bars_max)
sys.stdout.write(metrics.format(
'=' * bars + '-' * (bars_max - bars),
percent,
epoch_trained, FLAGS.num_per_epoch,
np.mean(batch_loss),
data_utils.time(time_spend), data_utils.time(time_estimate)
))
sys.stdout.flush()
if epoch_trained >= FLAGS.num_per_epoch:
break
print('\n')
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
model.saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_name))
def train():
"""训练模型"""
# 准备数据
print('准备数据')
#数据预处理有两步：1、decode_conv 2、data_utils
#原始数据集不是很好的问答式数据集。用decode_conv处理的数据，假定有ABC三个句子，则处理成两句问答：A:B,B:C，然后都插入到sqlite3里
#生成一个conversion.db文件，然后使用data_utils来进行语句处理，即对这个db文件做进一步处理
#对应四种格式，5_15,10_20,15_25,20_30,分别代表问句和回答句的字数上限。比如5_15即问句不超过5个字且答句不超过15个字。
#这种方式也和命名实体识别的一个性质，是为了能最小padding，进行局部padding，如果有句子太长的，但是不太多，那么可以滤掉。
#因为一般的对话都不会太长
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有数据 {} 条'.format(total_size))
#到这里为止还只是拿到四个bucket里的数据,并统计了一下总的数据条数
# 开始建模与训练
with tf.Session() as sess:
#整体流程即：1、创建模型 2、接收数据，并转换成模型可接收的类型 3、放入模型，计算损失 4、更新参数
# 构建模型
model = create_model(sess, False)
# 初始化变量
sess.run(tf.global_variables_initializer())
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
] #i=0,1,2,3==>bucket_sizes[: 1],
# 开始训练
metrics = ' '.join(
['\r[{}]', '{:.1f}%', '{}/{}', 'loss={:.3f}', '{}/{}'])
bars_max = 20
with tf.device('/gpu:0'):
for epoch_index in range(1, FLAGS.num_epoch + 1600):
print('Epoch {}:'.format(epoch_index))
time_start = time.time()
epoch_trained = 0
batch_loss = []
while True:
# 选择一个要训练的bucket
random_number = np.random.random_sample()
#tmp=[]
#for i in range(len(buckets_scale)):
#if buckets_scale[i] > random_number:
#tmp.append(i)
#bucket_id = min(tmp)
bucket_id = 1 if random_number <= 0.25 else 2 if random_number > 0.25 and random_number <= 0.5 else 3 if random_number > 0.5 and random_number < 0.75 else 4
bucket_id -= 1
#先选择对应的问答对长度，因为后面无论是padding还是生结果，都是根据这个位数来的
#bucket_id = min([i for i in range(len(buckets_scale)) if buckets_scale[i] > random_number])
#拿出64个问答对，data 和data_in 问答倒转
data, data_in = model.get_batch_data(
bucket_dbs, bucket_id) #先获取到问答对和答问对
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_dbs, bucket_id, data
) #再得到padding后的encoder_inputs,decoder_inputs和新生成的权重decoder_weights
#而这里的encoder_inputs，decoder_inputs都只是对应的字ID信息，而decoder_weights则是1和0组成的,也是和字位置一对一对应
#通过源码可以看出，ID只是初步信息，随机初始化一个embedding是embedding_attention_seq2seq内部会有的
_, step_loss, output = model.step(
sess, encoder_inputs, decoder_inputs, decoder_weights,
bucket_id, False
) #给定需要喂入的参数，即encoder、decoder、weights以及选择的bucket_id
#根据训练和测试状态，获取输出结果。
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss) #为了计算损失用
time_now = time.time()
time_spend = time_now - time_start
time_estimate = time_spend / (epoch_trained /
FLAGS.num_per_epoch)
percent = min(100,
epoch_trained / FLAGS.num_per_epoch) * 100
bars = math.floor(percent / 100 * bars_max)
sys.stdout.write(
metrics.format('=' * bars + '-' * (bars_max - bars),
percent,
epoch_trained, FLAGS.num_per_epoch,
np.mean(batch_loss),
data_utils.time(time_spend),
data_utils.time(time_estimate)))
sys.stdout.flush()
if epoch_trained >= FLAGS.num_per_epoch:
break
print('\n')
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
if epoch_index % 800 == 0:
model.saver.save(sess,
os.path.join(FLAGS.model_dir, FLAGS.model_name))
def test_bleu(count):
"""测试bleu"""
from nltk.translate.bleu_score import sentence_bleu
from tqdm import tqdm
# 准备数据
print('准备数据')
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有数据 {} 条'.format(total_size))
# bleu设置0的话，默认对所有样本采样
if count <= 0:
count = total_size
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
]
with tf.Session() as sess:
# 构建模型
model = create_model(sess, True)
model.batch_size = 1
# 初始化变量
sess.run(tf.initialize_all_variables())
model.saver.restore(sess, os.path.join(FLAGS.model_dir, FLAGS.model_name))
total_score = 0.0
for i in tqdm(range(count)):
# 选择一个要训练的bucket
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if buckets_scale[i] > random_number
])
data, _ = model.get_batch_data(
bucket_dbs,
bucket_id
)
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_dbs,
bucket_id,
data
)
_, _, output_logits = model.step(
sess,
encoder_inputs,
decoder_inputs,
decoder_weights,
bucket_id,
True
)
outputs = [int(np.argmax(logit, axis=1)) for logit in output_logits]
ask, _ = data[0]
all_answers = bucket_dbs[bucket_id].all_answers(ask)
ret = data_utils.indice_sentence(outputs)
if not ret:
continue
references = [list(x) for x in all_answers]
score = sentence_bleu(
references,
list(ret),
weights=(1.0,)
)
total_score += score
print('BLUE: {:.2f} in {} samples'.format(total_score / count * 10, count))
def train():
"""训练模型"""
# 准备数据
print('准备数据')
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有数据 {} 条'.format(total_size))
# 开始建模与训练
with tf.Session() as sess:
# 构建模型
model = create_model(sess, False)
# 初始化变量
sess.run(tf.initialize_all_variables())
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
]
# 开始训练
metrics = ' '.join([
'\r[{}]',
'{:.1f}%',
'{}/{}',
'loss={:.3f}',
'{}/{}'
])
bars_max = 20
for epoch_index in range(1, FLAGS.num_epoch + 1):
print('Epoch {}:'.format(epoch_index))
time_start = time.time()
epoch_trained = 0
batch_loss = []
while True:
# 选择一个要训练的bucket
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if buckets_scale[i] > random_number
])
data, data_in = model.get_batch_data(
bucket_dbs,
bucket_id
)
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_dbs,
bucket_id,
data
)
_, step_loss, output = model.step(
sess,
encoder_inputs,
decoder_inputs,
decoder_weights,
bucket_id,
False
)
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss)
time_now = time.time()
time_spend = time_now - time_start
time_estimate = time_spend / (epoch_trained / FLAGS.num_per_epoch)
percent = min(100, epoch_trained / FLAGS.num_per_epoch) * 100
bars = math.floor(percent / 100 * bars_max)
sys.stdout.write(metrics.format(
'=' * bars + '-' * (bars_max - bars),
percent,
epoch_trained, FLAGS.num_per_epoch,
np.mean(batch_loss),
data_utils.time(time_spend), data_utils.time(time_estimate)
))
sys.stdout.flush()
if epoch_trained >= FLAGS.num_per_epoch:
break
print('\n')
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
model.saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_name))
def train():
"""訓練模型"""
# 准备数据
print('準備數據')
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有數據 {} 條'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有數據 {} 條'.format(total_size))
# 開始建模
with tf.Session() as sess:
# 構建模型
model = create_model(sess, False)
# 初始化變量
sess.run(tf.initialize_all_variables())
buckets_scale = [
sum(bucket_sizes[:i + 1]) / total_size
for i in range(len(bucket_sizes))
]
# 開始訓練
metrics = ' '.join(
['\r[{}]', '{:.1f}%', '{}/{}', 'loss={:.3f}', '{}/{}'])
bars_max = 20
for epoch_index in range(1, FLAGS.num_epoch + 1):
print('Epoch {}:'.format(epoch_index))
time_start = time.time()
epoch_trained = 0
batch_loss = []
while True:
# 選擇一個要訓練的bucket
random_number = np.random.random_sample()
bucket_id = min([
i for i in range(len(buckets_scale))
if buckets_scale[i] > random_number
])
data, data_in = model.get_batch_data(bucket_dbs, bucket_id)
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_dbs, bucket_id, data)
_, step_loss, output = model.step(sess, encoder_inputs,
decoder_inputs,
decoder_weights, bucket_id,
False)
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss)
time_now = time.time()
time_spend = time_now - time_start
time_estimate = time_spend / (epoch_trained /
FLAGS.num_per_epoch)
percent = min(100, epoch_trained / FLAGS.num_per_epoch) * 100
bars = math.floor(percent / 100 * bars_max)
sys.stdout.write(
metrics.format('=' * bars + '-' * (bars_max - bars),
percent, epoch_trained, FLAGS.num_per_epoch,
np.mean(batch_loss),
data_utils.time(time_spend),
data_utils.time(time_estimate)))
sys.stdout.flush()
if epoch_trained >= FLAGS.num_per_epoch:
break
print('\n')
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
model.saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_name))
def train():
"""训练模型"""
print('数据准备中...')
bucket_dbs = data_utils.read_bucket_dbs(FLAGS.buckets_dir)
bucket_sizes = []
for i in range(len(buckets)):
bucket_size = bucket_dbs[i].size
bucket_sizes.append(bucket_size)
print('bucket {} 中有数据 {} 条'.format(i, bucket_size))
total_size = sum(bucket_sizes)
print('共有数据 {} 条'.format(total_size))
with tf.Session() as sess:
model = create_model(sess, False)
sess.run(tf.global_variables_initializer())
# 计算每个文件数据占比
buckets_scale = [sum(bucket_sizes[:i + 1]) / total_size for i in range(len(bucket_sizes))]
# 格式化控制台输出
metrics = ' '.join([
'\r[{}]',
'{:.1f}%',
'{}/{}',
'loss={:.3f}',
'{}/{}'
])
bars_max = 20
with tf.device('/gpu:0'):
for epoch_index in range(1, FLAGS.num_epoch + 1600):
print('Epoch {}:'.format(epoch_index))
time_start = time.time()
epoch_trained = 0 # 每个epoch已经训练的样本数
batch_loss = []
while True:
# 随机选择一个要训练的bucket_id
random_number = np.random.random_sample()
bucket_id = min([i for i in range(len(buckets_scale)) if buckets_scale[i] > random_number])
# 拿出64个问答对 data, data_in 问答倒转
data, data_in = model.get_batch_data(
bucket_dbs,
bucket_id
)
# 将问答对转换为模型训练可接受的格式
# bucket_10_20这个bucket对应的维度为：10*64 20*64 20*64
encoder_inputs, decoder_inputs, decoder_weights = model.get_batch(
bucket_dbs,
bucket_id,
data
)
# 训练
_, step_loss, output = model.step(
sess,
encoder_inputs,
decoder_inputs,
decoder_weights,
bucket_id,
False
)
epoch_trained += FLAGS.batch_size
batch_loss.append(step_loss)
time_now = time.time()
time_spend = time_now - time_start
time_estimate = time_spend / (epoch_trained / FLAGS.num_per_epoch)
percent = min(100, epoch_trained / FLAGS.num_per_epoch) * 100
bars = math.floor(percent / 100 * bars_max)
sys.stdout.write(metrics.format(
'=' * bars + '-' * (bars_max - bars),
percent,
epoch_trained, FLAGS.num_per_epoch,
np.mean(batch_loss),
data_utils.time(time_spend), data_utils.time(time_estimate)
))
sys.stdout.flush()
if epoch_trained >= FLAGS.num_per_epoch:
break
print('\n')
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
if epoch_index%800==0:
model.saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_name))
