def fit(self, x_train, y_train, x_dev, y_dev):
"""
训练
:param x_train:
:param y_train:
:param x_dev:
:param y_dev:
:return:
"""
# 保存超参数
self.hyper_parameters["model"]["is_training"] = False # 预测时候这些设为False
self.hyper_parameters["model"]["trainable"] = False
self.hyper_parameters["model"]["dropout"] = 1.0
save_json(json_lines=self.hyper_parameters,
json_path=self.path_hyper_parameters)
# 训练模型
self.model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.epochs,
validation_data=(x_dev, y_dev),
shuffle=True,
callbacks=self.callback())
# 保存embedding, 动态的
if self.trainable:
self.word_embedding.model.save(self.path_fineture)
def fit_generator(self, embed, rate=1, encoding="utf-8"):
"""
:param data_fit_generator: yield, 训练数据
:param data_dev_generator: yield, 验证数据
:param steps_per_epoch: int, 训练一轮步数
:param validation_steps: int, 验证一轮步数
:return:
"""
# 保存超参数
self.hyper_parameters["model"]["is_training"] = False # 预测时候这些设为False
self.hyper_parameters["model"]["trainable"] = False
self.hyper_parameters["model"]["dropout"] = 0.0
save_json(json_lines=self.hyper_parameters,
json_path=self.path_hyper_parameters)
pg = PreprocessGenerator(self.path_model_l2i_i2l)
_, len_train = pg.preprocess_label2set(
self.hyper_parameters["data"]["train_data"],
self.embedding_type,
encoding=encoding)
data_fit_generator = pg.preprocess_label_question_to_idx_fit_generator(
embedding_type=self.hyper_parameters["embedding_type"],
crf_mode=self.hyper_parameters["model"]["crf_mode"],
path=self.hyper_parameters["data"]["train_data"],
batch_size=self.batch_size,
embed=embed,
rate=rate,
epochs=self.epochs,
encoding=encoding)
_, len_val = pg.preprocess_label2set(
self.hyper_parameters["data"]["val_data"],
self.embedding_type,
encoding=encoding)
data_dev_generator = pg.preprocess_label_question_to_idx_fit_generator(
embedding_type=self.hyper_parameters["embedding_type"],
crf_mode=self.hyper_parameters["model"]["crf_mode"],
path=self.hyper_parameters["data"]["val_data"],
batch_size=self.batch_size,
embed=embed,
rate=rate,
epochs=self.epochs,
encoding=encoding)
steps_per_epoch = (int(len_train * rate) if len_train > 500 else
len_train) // self.batch_size + 1
validation_steps = (int(len_val * rate) if len_val > 500 else
len_val) // self.batch_size + 1
# 训练模型
self.model.fit_generator(generator=data_fit_generator,
validation_data=data_dev_generator,
callbacks=self.callback(),
epochs=self.epochs,
steps_per_epoch=steps_per_epoch,
validation_steps=validation_steps,
shuffle=True)
# 保存embedding, 动态的
if self.trainable:
self.word_embedding.model.save(self.path_fineture)
def build(self, **kwargs):
self.vocab_size = len(self.token2idx)
self.input = tf.keras.layers.Input(shape=(self.len_max,), dtype='int32')
self.output = tf.keras.layers.Embedding(self.vocab_size,
self.embed_size,
input_length=self.len_max,
trainable=self.trainable,
)(self.input)
self.model = tf.keras.Model(self.input, self.output)
save_json(json_lines=self.token2idx, json_path=os.path.join(self.path_model_dir, 'vocab.txt'))
def save_delete_words(self, words):
"""
删除词语到用户词典, 可持久化, 重载有效
:param word_freqs: list, like ['大漠帝国']
:return: None
"""
assert type(words) == list
for w in words:
self.delete_word(w) # 删除到总字典, 不持久化
if w in self.dict_user: self.dict_user.pop(w) # 删除到用户字典, 持久化
save_json([self.dict_user], path_dict_user)
def save_add_words(self, word_freqs):
"""
新增词语到用户词典, 可持久化, 重载有效
:param word_freqs: dict, like {'大漠帝国':132}
:return: None
"""
assert type(word_freqs) == dict
for k, v in word_freqs.items():
self.add_word(k, v) # 新增到总字典, 不持久化
self.dict_user[k] = v # 新增到用户字典, 持久化
save_json([self.dict_user], path_dict_user)
def build(self, **kwargs):
self.vocab_size = len(self.token2idx)
logger.info("vocab_size is {}".format(str(self.vocab_size)))
self.input = tf.keras.layers.Input(shape=(self.len_max, ),
dtype='int32',
name="input")
self.output = tf.keras.layers.Embedding(
self.vocab_size + 1,
self.embed_size,
input_length=self.len_max,
trainable=self.trainable,
name="embedding_{}".format(str(self.embed_size)))(self.input)
self.model = tf.keras.Model(self.input, self.output)
save_json(json_lines=self.token2idx,
json_path=os.path.join(self.path_model_dir, 'vocab.txt'))
def build(self, **kwargs):
self.embedding_type = 'word2vec'
# logger.info("load word2vec start!")
self.key_vector = KeyedVectors.load_word2vec_format(
self.corpus_path, **kwargs)
# logger.info("load word2vec end!")
self.embed_size = self.key_vector.vector_size
self.token2idx = self.ot_dict.copy()
embedding_matrix = []
# 首先加self.token2idx中的四个[PAD]、[UNK]、[BOS]、[EOS]
embedding_matrix.append(np.zeros(self.embed_size))
embedding_matrix.append(np.random.uniform(-0.5, 0.5, self.embed_size))
embedding_matrix.append(np.random.uniform(-0.5, 0.5, self.embed_size))
embedding_matrix.append(np.random.uniform(-0.5, 0.5, self.embed_size))
for word in self.key_vector.index2entity:
self.token2idx[word] = len(self.token2idx)
embedding_matrix.append(self.key_vector[word])
# self.token2idx = self.token2idx
self.idx2token = {}
for key, value in self.token2idx.items():
self.idx2token[value] = key
self.vocab_size = len(self.token2idx)
logger.info("vocab_size is {}".format(str(self.vocab_size)))
embedding_matrix = np.array(embedding_matrix)
# self.input = Input(shape=(self.len_max,), dtype='int32')
self.input = tf.keras.layers.Input(shape=(self.len_max, ),
dtype='int32',
name="input")
self.output = tf.keras.layers.Embedding(
self.vocab_size,
self.embed_size,
input_length=self.len_max,
weights=[embedding_matrix],
trainable=self.trainable,
name="embedding_{}".format(str(self.embed_size)))(self.input)
self.model = tf.keras.Model(self.input, self.output)
# 保存字/词典
save_json(json_lines=self.token2idx,
json_path=os.path.join(self.path_model_dir, 'vocab.txt'))
def preprocess_label_question_to_idx_fit(self,
embedding_type,
path,
embed,
rate=1,
crf_mode='reg'):
"""
fit用, 关键:对每一条数据操作，获取label和问句index
:param embedding_type: str, like 'albert'
:param path: str, like 'train.json'
:param embed: class, like embed
:param rate: float, like 0.9
:param crf_mode: str, like 'reg', 'pad'
:return: np.array
"""
# 首先获取label,set,即存在的具体类
label_set, len_all = self.preprocess_label2set(path, embedding_type)
# 获取label转index字典等, 如果label2index存在则不转换了, dev验证集合的时候用
if not os.path.exists(self.path_model_l2i_i2l):
count = 0
label2index = {}
index2label = {}
for label_one in label_set:
label2index[label_one] = count
index2label[count] = label_one
count = count + 1
l2i_i2l = {}
l2i_i2l['l2i'] = label2index
l2i_i2l['i2l'] = index2label
save_json(l2i_i2l, self.path_model_l2i_i2l)
else:
l2i_i2l = load_json(self.path_model_l2i_i2l)
# 读取数据的比例
len_ql = int(rate * len_all)
if len_ql <= 500: # sample时候不生效,使得语料足够训练
len_ql = len_all
def process_line(line, embed, l2i_i2l):
"""
对每一条数据操作，获取label和问句index
:param line:
:param embed:
:param l2i_i2l:
:return:
"""
# 对每一条数据操作，对question和label进行padding
ques_label = json.loads(line.strip())
label_org = ques_label["label"]
label_index = [l2i_i2l["l2i"][lr] for lr in label_org]
# len_sequence = len(label_index)
que_embed = embed.sentence2idx("".join(ques_label["question"]))
# label padding
if embedding_type in ['bert', 'albert']:
# padding label
len_leave = embed.len_max - len(label_index) - 2
if len_leave >= 0:
label_index_leave = [l2i_i2l["l2i"]["<CLS>"]] + [
li for li in label_index
] + [l2i_i2l["l2i"]["<PAD>"] for _ in range(len_leave)
] + [l2i_i2l["l2i"]["<SEP>"]]
else:
label_index_leave = [
l2i_i2l["l2i"]["<CLS>"]
] + label_index[0:embed.len_max -
2] + [l2i_i2l["l2i"]["<SEP>"]]
else:
# padding label
len_leave = embed.len_max - len(label_index) # -2
if len_leave >= 0:
label_index_leave = [li for li in label_index] + [
l2i_i2l["l2i"]["<PAD>"] for i in range(len_leave)
]
else:
label_index_leave = label_index[0:embed.len_max]
# 转为one-hot
label_res = to_categorical(label_index_leave,
num_classes=len(l2i_i2l["l2i"]))
return que_embed, label_res
file_csv = open(path, "r", encoding="utf-8")
cout_all_line = 0
cnt = 0
x, y = [], []
for line in file_csv:
# 跳出循环
if len_ql < cout_all_line:
break
cout_all_line += 1
if line.strip():
# 一个json一个json处理
# 备注:最好训练前先处理,使得ques长度小于等于len_max(word2vec), len_max-2(bert, albert)
x_line, y_line = process_line(line, embed, l2i_i2l)
x.append(x_line)
y.append(y_line.tolist())
cnt += 1
# 通过两种方式处理: 1.嵌入类型(bert, word2vec, random), 2.条件随机场(CRF:'pad', 'reg')类型
if embedding_type in ['bert', 'albert']:
x_, y_ = np.array(x), np.array(y)
x_1 = np.array([x[0] for x in x_])
x_2 = np.array([x[1] for x in x_])
x_3 = np.array([x[2] for x in x_])
if crf_mode == 'pad':
x_all = [x_1, x_2, x_3]
elif crf_mode == 'reg':
x_all = [x_1, x_2]
else:
x_all = [x_1, x_2]
else:
x_, y_ = np.array(x), np.array(y)
x_1 = np.array([x[0] for x in x_])
x_2 = np.array([x[1] for x in x_])
if crf_mode == 'pad':
x_all = [x_1, x_2]
elif crf_mode == 'reg':
x_all = x_1
else:
x_all = x_1
# 使用fit的时候, return返回
return x_all, y_