def main(_):
model_path = os.path.join('models', Config.file_name)
vocab_file = os.path.join(model_path, 'vocab_tuples.pkl')
# 获取测试问题
sens_tags_test = get_sens_tags('data/test.txt')
# 数据处理
converter = TextConverter(None,
vocab_file,
max_vocab=Config.vocab_max_size)
print('vocab size:', converter.vocab_size)
# 产生测试样本
test_QA_arrs = converter.QAs_to_arr(sens_tags_test, Config.seq_length)
# 加载上一次保存的模型
model = Model(Config, converter.vocab_size)
checkpoint_path = tf.train.latest_checkpoint(model_path)
if checkpoint_path:
model.load(checkpoint_path)
# 测试
print('start to testing...')
n = len(test_QA_arrs)
for i in range(n):
y_pre, y_cos = model.test(test_QA_arrs[i])
tags = [converter.int_to_tag(id) for id in y_pre[:test_QA_arrs[i][2]]]
print('\nword / tag / pre')
for j in range(test_QA_arrs[i][2]):
print("{} / {} / {}".format(sens_tags_test[i][0][j],
sens_tags_test[i][1][j], tags[j]))
def main(_):
model_path = os.path.join('models', Config.file_name)
if os.path.exists(model_path) is False:
os.makedirs(model_path)
# train_file = 'data/train.txt'
# train_file = 'data/train.0926.label'
# dev_file = 'data/slotlabel3400'
train_file = sys.argv[1]
dev_file = sys.argv[2]
save_file = os.path.join(model_path, 'vocab_tuples.pkl')
# 获取样本数据
sens_tags_train = get_sens_tags(train_file)
sens_tags_val = get_sens_tags(dev_file)
# sens_tags = get_sens_tags(train_file)
#
# sens_tags_train = sens_tags[10000:]
# sens_tags_val = sens_tags[:10000]
# 数据处理
converter = TextConverter(train_file,
save_file,
max_vocab=Config.vocab_max_size)
print('vocab size:', converter.vocab_size)
Config.num_classes = converter.tag_size + 1
# 产生训练样本
train_QA_arrs = converter.QAs_to_arr(sens_tags_train, Config.seq_length)
train_g = converter.batch_generator(train_QA_arrs, Config.batch_size)
# 产生验证样本
val_QA_arrs = converter.QAs_to_arr(sens_tags_val, Config.seq_length)
val_g = converter.val_samples_generator(val_QA_arrs, Config.batch_size)
# 加载上一次保存的模型
model = Model(Config, converter.vocab_size)
checkpoint_path = tf.train.latest_checkpoint(model_path)
if checkpoint_path:
model.load(checkpoint_path)
print('start to training...')
model.train(train_g, model_path, val_g)
def main(_):
model_path = os.path.join('models', Config.file_name)
if os.path.exists(model_path) is False:
os.makedirs(model_path)
input_file = 'data/去除2和null.xlsx'
vocab_file = os.path.join(model_path, 'vocab_label.pkl')
# 获取原始excel数据
QAs = get_excel_QAs(
input_file) # 要求excel文件格式，第一个表，第一列id，第二列query,第三列response
# 分配训练和验证数据集
thres = int(0.8 * len(QAs))
train_QAs = QAs[:thres]
val_QAs = QAs[thres:]
# 数据处理
text = get_QAs_text(train_QAs)
converter = TextConverter(text,
vocab_file,
max_vocab=Config.vocab_max_size,
seq_length=Config.seq_length)
print('vocab size:', converter.vocab_size)
# 产生训练样本
train_QA_arrs = converter.QAs_to_arr(train_QAs)
train_g = converter.batch_generator(train_QA_arrs, Config.batch_size)
# 产生验证样本
val_QA_arrs = converter.QAs_to_arr(val_QAs)
val_g = converter.val_samples_generator(val_QA_arrs, Config.batch_size)
# 加载上一次保存的模型
model = Model(Config, converter.vocab_size)
checkpoint_path = tf.train.latest_checkpoint(model_path)
if checkpoint_path:
model.load(checkpoint_path)
print('start to training...')
model.train(train_g, model_path, val_g)
