def inference(self,
request: str,
beam_size: int,
start_sign: str = "<start>",
end_sign: str = "<end>") -> AnyStr:
""" 对话推断模块
:param request: 输入句子
:param beam_size: beam大小
:param start_sign: 句子开始标记
:param end_sign: 句子结束标记
:return: 返回历史指标数据
"""
tokenizer = load_tokenizer(self.dict_path)
enc_input = preprocess_request(sentence=request,
tokenizer=tokenizer,
max_length=self.max_sentence,
start_sign=start_sign,
end_sign=end_sign)
enc_output, padding_mask = self.encoder(inputs=enc_input)
dec_input = tf.expand_dims([tokenizer.word_index.get(start_sign)], 0)
beam_search_container = BeamSearch(beam_size=beam_size,
max_length=self.max_sentence,
worst_score=0)
beam_search_container.reset(enc_output=enc_output,
dec_input=dec_input,
remain=padding_mask)
enc_output, dec_input, padding_mask = beam_search_container.get_search_inputs(
)
for t in range(self.max_sentence):
predictions = self._inference_one_step(dec_input=dec_input,
enc_output=enc_output,
padding_mask=padding_mask)
beam_search_container.expand(
predictions=predictions,
end_sign=tokenizer.word_index.get(end_sign))
# 注意了，如果BeamSearch容器里的beam_size为0了，说明已经找到了相应数量的结果，直接跳出循环
if beam_search_container.beam_size == 0:
break
enc_output, dec_input, padding_mask = beam_search_container.get_search_inputs(
)
beam_search_result = beam_search_container.get_result(top_k=3)
result = ""
# 从容器中抽取序列，生成最终结果
for i in range(len(beam_search_result)):
temp = beam_search_result[i].numpy()
text = tokenizer.sequences_to_texts(temp)
text[0] = text[0].replace(start_sign,
"").replace(end_sign,
"").replace(" ", "")
result = "<" + text[0] + ">" + result
return result
def inference(self,
request: list,
solr: pysolr.Solr,
max_utterance: int,
d_type: tf.dtypes.DType = tf.float32,
*args,
**kwargs) -> AnyStr:
""" 对话推断模块
:param request: 输入对话历史
:param solr: solr服务
:param max_utterance: 每轮最大语句数
:param d_type: 运算精度
:return: 返回历史指标数据
"""
tokenizer = load_tokenizer(self.dict_path)
history = request[-max_utterance:]
pad_sequences = [0] * self.max_sentence
utterance = tokenizer.texts_to_sequences(history)
utterance_len = len(utterance)
# 如果当前轮次中的历史语句不足max_utterances数量，需要在尾部进行填充
if utterance_len != max_utterance:
utterance = [pad_sequences
] * (max_utterance - utterance_len) + utterance
utterance = tf.keras.preprocessing.sequence.pad_sequences(
sequences=utterance, maxlen=self.max_sentence, padding="post")
tf_idf = get_tf_idf_top_k(history=history, k=5)
query = "{!func}sum("
for key in tf_idf:
query += "product(idf(utterance," + key + "),tf(utterance," + key + ")),"
query += ")"
candidates = solr.search(q=query, start=0, rows=10).docs
candidates = [candidate["utterance"][0] for candidate in candidates]
if candidates is None:
return "Sorry! I didn't hear clearly, can you say it again?"
else:
utterances = [utterance] * len(candidates)
responses = tokenizer.texts_to_sequences(candidates)
responses = tf.keras.preprocessing.sequence.pad_sequences(
sequences=responses, maxlen=self.max_sentence, padding="post")
utterances = tf.convert_to_tensor(value=utterances)
responses = tf.convert_to_tensor(value=responses)
index = self._inference_one_step(utterances=utterances,
responses=responses)
return candidates[index]
def inference(self,
request: str,
beam_size: int,
start_sign: str = "<start>",
end_sign: str = "<end>") -> AnyStr:
""" 对话推断模块
:param request: 输入句子
:param beam_size: beam大小
:param start_sign: 句子开始标记
:param end_sign: 句子结束标记
:return: 返回历史指标数据
"""
with torch.no_grad():
tokenizer = load_tokenizer(self.dict_path)
enc_input = preprocess_request(sentence=request,
tokenizer=tokenizer,
max_length=self.max_sentence,
start_sign=start_sign,
end_sign=end_sign)
enc_input = torch.tensor(data=enc_input,
dtype=torch.long).permute(1, 0)
enc_output, states = self.encoder(inputs=enc_input)
dec_input = torch.tensor(
data=[[tokenizer.word_index.get(start_sign)]])
beam_search_container = BeamSearch(beam_size=beam_size,
max_length=self.max_sentence,
worst_score=0)
beam_search_container.reset(enc_output=enc_output.permute(1, 0, 2),
dec_input=dec_input,
remain=states)
enc_output, dec_input, states = beam_search_container.get_search_inputs(
)
enc_output = enc_output.permute(1, 0, 2)
for t in range(self.max_sentence):
predictions, dec_hidden = self.decoder(dec_input, states,
enc_output)
predictions = F.softmax(input=predictions, dim=-1)
beam_search_container.expand(
predictions=predictions[0],
end_sign=tokenizer.word_index.get(end_sign))
if beam_search_container.beam_size == 0:
break
enc_output, dec_input, states = beam_search_container.get_search_inputs(
)
dec_input = dec_input[:, -1].unsqueeze(-1)
enc_output = enc_output.permute(1, 0, 2)
dec_input = dec_input.permute(1, 0)
beam_search_result = beam_search_container.get_result(top_k=3)
result = ""
# 从容器中抽取序列，生成最终结果
for i in range(len(beam_search_result)):
temp = beam_search_result[i].numpy()
text = tokenizer.sequences_to_texts(temp)
text[0] = text[0].replace(start_sign,
"").replace(end_sign,
"").replace(" ", "")
result = "<" + text[0] + ">" + result
return result
def load_data(dict_path: str, buffer_size: int, batch_size: int, train_data_type: str, valid_data_type: str,
max_sentence: int, valid_data_split: float = 0.0, train_data_path: str = "", valid_data_path: str = "",
max_train_data_size: int = 0, max_valid_data_size: int = 0, **kwargs) -> Tuple:
""" 数据加载方法
:param dict_path: 字典路径
:param buffer_size: Dataset加载缓存大小
:param batch_size: Dataset加载批大小
:param train_data_type: 读取训练数据类型，单轮/多轮...
:param valid_data_type: 读取验证数据类型，单轮/多轮...
:param max_sentence: 单个句子最大长度
:param valid_data_split: 用于从训练数据中划分验证数据
:param train_data_path: 文本数据路径
:param valid_data_path: 验证数据文本路径
:param max_train_data_size: 最大训练数据量
:param max_valid_data_size: 最大验证数据量
:return: 训练Dataset、验证Dataset、训练数据总共的步数、验证数据总共的步数和检查点前缀
"""
tokenizer = load_tokenizer(dict_path=dict_path)
train_flag = True # 是否开启训练标记
train_steps_per_epoch = 0
train_first, train_second, train_third = None, None, None
valid_flag = True # 是否开启验证标记
valid_steps_per_epoch = 0
valid_first, valid_second, valid_third = None, None, None
if train_data_path != "":
train_first, train_second, train_third = read_data(
data_path=train_data_path, max_data_size=max_train_data_size,
max_sentence=max_sentence, data_type=train_data_type, tokenizer=tokenizer, **kwargs
)
else:
train_flag = False
if valid_data_path != "":
print("读取验证对话对...")
valid_first, valid_second, valid_third = read_data(
data_path=valid_data_path, max_data_size=max_valid_data_size,
max_sentence=max_sentence, data_type=valid_data_type, tokenizer=tokenizer, **kwargs
)
elif valid_data_split != 0.0:
train_size = int(len(train_first) * (1.0 - valid_data_split))
valid_first = train_first[train_size:]
valid_second = train_second[train_size:]
valid_third = train_third[train_size:]
train_first = train_first[:train_size]
train_second = train_second[:train_size]
train_third = train_third[:train_size]
else:
valid_flag = False
if train_flag:
train_dataset = tf.data.Dataset.from_tensor_slices((train_first, train_second, train_third)).cache().shuffle(
buffer_size, reshuffle_each_iteration=True).prefetch(tf.data.experimental.AUTOTUNE)
train_dataset = train_dataset.batch(batch_size, drop_remainder=True)
train_steps_per_epoch = len(train_first) // batch_size
else:
train_dataset = None
if valid_flag:
valid_dataset = tf.data.Dataset.from_tensor_slices((valid_first, valid_second, valid_third)) \
.prefetch(tf.data.experimental.AUTOTUNE)
valid_dataset = valid_dataset.batch(batch_size, drop_remainder=True)
valid_steps_per_epoch = len(valid_first) // batch_size
else:
valid_dataset = None
return train_dataset, valid_dataset, train_steps_per_epoch, valid_steps_per_epoch
def load_data(dict_path: str,
batch_size: int,
train_data_type: str,
valid_data_type: str,
max_sentence: int,
valid_data_split: float = 0.0,
train_data_path: str = "",
valid_data_path: str = "",
max_train_data_size: int = 0,
max_valid_data_size: int = 0,
num_workers: int = 2,
**kwargs) -> Tuple:
""" 数据加载方法
:param dict_path: 字典路径
:param batch_size: Dataset加载批大小
:param train_data_type: 读取训练数据类型，单轮/多轮...
:param valid_data_type: 读取验证数据类型，单轮/多轮...
:param max_sentence: 单个句子最大长度
:param valid_data_split: 用于从训练数据中划分验证数据
:param train_data_path: 文本数据路径
:param valid_data_path: 验证数据文本路径
:param max_train_data_size: 最大训练数据量
:param max_valid_data_size: 最大验证数据量
:param num_workers: 数据加载器的工作线程
:return: 训练Dataset、验证Dataset、训练数据总共的步数、验证数据总共的步数和检查点前缀
"""
tokenizer = load_tokenizer(dict_path=dict_path)
train_flag = True # 是否开启训练标记
train_steps_per_epoch = 0
train_first, train_second, train_third = None, None, None
valid_flag = True # 是否开启验证标记
valid_steps_per_epoch = 0
valid_first, valid_second, valid_third = None, None, None
if train_data_path != "":
train_first, train_second, train_third = read_data(
data_path=train_data_path,
max_data_size=max_train_data_size,
max_sentence=max_sentence,
data_type=train_data_type,
tokenizer=tokenizer,
**kwargs)
else:
train_flag = False
if valid_data_path != "":
print("读取验证对话对...")
valid_first, valid_second, valid_third = read_data(
data_path=valid_data_path,
max_data_size=max_valid_data_size,
max_sentence=max_sentence,
data_type=valid_data_type,
tokenizer=tokenizer,
**kwargs)
elif valid_data_split != 0.0:
train_size = int(len(train_first) * (1.0 - valid_data_split))
valid_first = train_first[train_size:]
valid_second = train_second[train_size:]
valid_third = train_third[train_size:]
train_first = train_first[:train_size]
train_second = train_second[:train_size]
train_third = train_third[:train_size]
else:
valid_flag = False
if train_flag:
train_dataset = PairDataset(train_first, train_second, train_third)
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True,
drop_last=True,
num_workers=num_workers)
train_steps_per_epoch = len(train_first) // batch_size
else:
train_loader = None
if valid_flag:
valid_dataset = PairDataset(valid_first, valid_second, valid_third)
valid_loader = DataLoader(dataset=valid_dataset,
batch_size=batch_size,
shuffle=False,
drop_last=True,
num_workers=num_workers)
valid_steps_per_epoch = len(valid_first) // batch_size
else:
valid_loader = None
return train_loader, valid_loader, train_steps_per_epoch, valid_steps_per_epoch
def tf_seq2seq() -> NoReturn:
parser = ArgumentParser(description="seq2seq chatbot")
parser.add_argument("--version", default="tf", type=str, required=True, help="执行版本")
parser.add_argument("--model", default="transformer", type=str, required=True, help="执行模型")
parser.add_argument("--config_file", default="", type=str, required=False, help="配置文件路径，为空则默认命令行，不为空则使用配置文件参数")
parser.add_argument("--act", default="pre_treat", type=str, required=False, help="执行类型")
parser.add_argument("--cell_type", default="lstm", type=str, required=False, help="rnn的cell类型")
parser.add_argument("--if_bidirectional", default=True, type=bool, required=False, help="是否开启双向rnn")
parser.add_argument("--units", default=1024, type=int, required=False, help="隐藏层单元数")
parser.add_argument("--vocab_size", default=20000, type=int, required=False, help="词汇大小")
parser.add_argument("--embedding_dim", default=512, type=int, required=False, help="嵌入层维度大小")
parser.add_argument("--encoder_layers", default=6, type=int, required=False, help="encoder的层数")
parser.add_argument("--decoder_layers", default=6, type=int, required=False, help="decoder的层数")
parser.add_argument("--max_train_data_size", default=0, type=int, required=False, help="用于训练的最大数据大小")
parser.add_argument("--max_valid_data_size", default=0, type=int, required=False, help="用于验证的最大数据大小")
parser.add_argument("--max_sentence", default=40, type=int, required=False, help="单个序列的最大长度")
parser.add_argument("--dict_path", default="data/preprocess/seq2seq_dict.json",
type=str, required=False, help="字典路径")
parser.add_argument("--checkpoint_dir", default="checkpoints/tensorflow/seq2seq",
type=str, required=False, help="检查点路径")
parser.add_argument("--resource_data_path", default="data/LCCC.json", type=str, required=False, help="原始数据集路径")
parser.add_argument("--tokenized_data_path", default="data/preprocess/lccc_tokenized.txt",
type=str, required=False, help="处理好的多轮分词数据集路径")
parser.add_argument("--preprocess_data_path", default="data/preprocess/single_tokenized.txt",
type=str, required=False, help="处理好的单轮分词数据集路径")
parser.add_argument("--valid_data_path", default="data/preprocess/single_tokenized.txt", type=str,
required=False, help="处理好的单轮分词验证评估用数据集路径")
parser.add_argument("--history_image_dir", default="data/history/seq2seq/", type=str, required=False,
help="数据指标图表保存路径")
parser.add_argument("--valid_freq", default=5, type=int, required=False, help="验证频率")
parser.add_argument("--checkpoint_save_freq", default=2, type=int, required=False, help="检查点保存频率")
parser.add_argument("--checkpoint_save_size", default=1, type=int, required=False, help="单轮训练中检查点保存数量")
parser.add_argument("--batch_size", default=64, type=int, required=False, help="batch大小")
parser.add_argument("--buffer_size", default=20000, type=int, required=False, help="Dataset加载缓冲大小")
parser.add_argument("--beam_size", default=3, type=int, required=False, help="BeamSearch的beam大小")
parser.add_argument("--valid_data_split", default=0.2, type=float, required=False, help="从训练数据集中划分验证数据的比例")
parser.add_argument("--epochs", default=5, type=int, required=False, help="训练步数")
parser.add_argument("--start_sign", default="<start>", type=str, required=False, help="序列开始标记")
parser.add_argument("--end_sign", default="<end>", type=str, required=False, help="序列结束标记")
parser.add_argument("--unk_sign", default="<unk>", type=str, required=False, help="未登录词")
parser.add_argument("--encoder_save_path", default="models/tensorflow/seq2seq/encoder", type=str,
required=False, help="Encoder的SaveModel格式保存路径")
parser.add_argument("--decoder_save_path", default="models/tensorflow/seq2seq/decoder", type=str,
required=False, help="Decoder的SaveModel格式保存路径")
options = parser.parse_args().__dict__
execute_type = options["act"]
if options["config_file"] != "":
with open(options["config_file"], "r", encoding="utf-8") as config_file:
options = json.load(config_file)
# 注意了有关路径的参数，以tensorflow目录下为基准配置
file_path = os.path.abspath(__file__)
work_path = file_path[:file_path.find("tensorflow")]
encoder = seq2seq.encoder(
vocab_size=options["vocab_size"], embedding_dim=options["embedding_dim"], num_layers=options["encoder_layers"],
enc_units=options["units"] // 2, cell_type=options["cell_type"], if_bidirectional=options["if_bidirectional"]
)
decoder = seq2seq.decoder(
vocab_size=options["vocab_size"], embedding_dim=options["embedding_dim"], dec_units=options["units"],
enc_units=options["units"], num_layers=options["decoder_layers"], cell_type=options["cell_type"]
)
loss_metric = tf.keras.metrics.Mean(name="loss_metric")
accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy(name="accuracy_metric")
optimizer = tf.optimizers.Adam(name="optimizer")
checkpoint_manager = load_checkpoint(
checkpoint_dir=work_path + options["checkpoint_dir"], execute_type=execute_type,
encoder=encoder, decoder=decoder, checkpoint_save_size=options["checkpoint_save_size"]
)
modules = Seq2SeqModule(
loss_metric=loss_metric, accuracy_metric=accuracy_metric, batch_size=options["batch_size"],
buffer_size=options["buffer_size"], max_sentence=options["max_sentence"], train_data_type="read_single_data",
valid_data_type="read_single_data", dict_path=work_path + options["dict_path"], encoder=encoder, decoder=decoder
)
if execute_type == "pre_treat":
preprocess_dataset(dataset_name="lccc", raw_data_path=work_path + options["resource_data_path"],
tokenized_data_path=work_path + options["tokenized_data_path"], remove_tokenized=True)
to_single_turn_dataset(tokenized_data_path=work_path + options["tokenized_data_path"],
dict_path=work_path + options["dict_path"], unk_sign=options["unk_sign"],
start_sign=options["start_sign"], end_sign=options["end_sign"],
max_data_size=options["max_train_data_size"], vocab_size=options["vocab_size"],
qa_data_path=work_path + options["preprocess_data_path"])
elif execute_type == "train":
history = {"train_accuracy": [], "train_loss": [], "valid_accuracy": [], "valid_loss": []}
tokenizer = load_tokenizer(dict_path=work_path + options["dict_path"])
history = modules.train(
optimizer=optimizer, epochs=options["epochs"], checkpoint=checkpoint_manager, history=history,
train_data_path=work_path + options["preprocess_data_path"], valid_data_path="",
checkpoint_save_freq=options["checkpoint_save_freq"], max_valid_data_size=options["max_valid_data_size"],
max_train_data_size=options["max_train_data_size"], valid_data_split=options["valid_data_split"],
start_sign=tokenizer.word_index.get(options["start_sign"]),
encoder_save_path=work_path + options["encoder_save_path"],
decoder_save_path=work_path + options["decoder_save_path"]
)
show_history(history=history, valid_freq=options["checkpoint_save_freq"],
save_dir=work_path + options["history_image_dir"])
elif execute_type == "evaluate":
tokenizer = load_tokenizer(dict_path=work_path + options["dict_path"])
modules.evaluate(
max_valid_data_size=options["max_valid_data_size"], valid_data_path=work_path + options["valid_data_path"],
start_sign=tokenizer.word_index.get(options["start_sign"])
)
elif execute_type == "chat":
print("Agent: 你好！结束聊天请输入ESC。")
while True:
request = input("User: "******"ESC":
print("Agent: 再见！")
exit(0)
response = modules.inference(request=request, beam_size=options["beam_size"],
start_sign=options["start_sign"], end_sign=options["end_sign"])
print("Agent: ", response)
else:
parser.error(message="")