def __init__(self):
self.model_dir = "./models//cws_2"
self.data = Data()
self.char_index = {
k: v
for v, k in enumerate(self.data.get_dictionary())
}
self.unknow_char_id = len(self.char_index)
self.io_sequence_size = 130
vocab_size = len(self.char_index) + 1
self.classnames = {
'O': 0,
'B': 1,
'I': 2,
'S': 3
} # B:开始， I：中间词汇， S：单个字的词
class_size = len(self.classnames) # brand,price,position,item,unknow
keep_prob = 0.5
learning_rate = 0.0005
trainable = True
self.batch_size = 128
with tf.variable_scope('ner_query'):
self.model = FenciNerCore(self.io_sequence_size, vocab_size,
class_size, keep_prob, learning_rate,
trainable)
def __init__(self):
self.model_dir = "./models//ner_1"
self.data = Data()
self.char_index = {
k: v
for v, k in enumerate(self.data.get_dictionary())
}
self.keep_prob = 1.0
self.is_training = False
# self.char_index = {' ': 0}
# self.load_dict()
self.unknow_char_id = len(self.char_index)
self.io_sequence_size = 150 # 70
self.vocab_size = len(self.char_index) + 1
self.batch_size = 64
self.ner_index_class = {
'O': 0,
'B': 1,
'I': 2,
'S': 3
} # B:开始， I：中间词汇， S：单个字的词
self.class_size = len(self.ner_index_class)
self.classids = {}
for key in self.ner_index_class.keys():
self.classids[self.ner_index_class[key]] = key
self.graph = tf.Graph()
with self.graph.as_default():
with tf.variable_scope('ner_query'):
self.model = FenciNerCore(
io_sequence_size=self.io_sequence_size,
vocab_size=self.vocab_size,
class_size=self.class_size,
keep_prob=self.keep_prob,
trainable=self.is_training)
self.saver = tf.train.Saver()
config = tf.ConfigProto()
self.session = tf.Session(graph=self.graph, config=config)
with self.session.as_default():
self.load()
class NerPredicter:
def __init__(self):
self.model_dir = "./models//ner_1"
self.data = Data()
self.char_index = {
k: v
for v, k in enumerate(self.data.get_dictionary())
}
self.keep_prob = 1.0
self.is_training = False
# self.char_index = {' ': 0}
# self.load_dict()
self.unknow_char_id = len(self.char_index)
self.io_sequence_size = 150 # 70
self.vocab_size = len(self.char_index) + 1
self.batch_size = 64
self.ner_index_class = {
'O': 0,
'B': 1,
'I': 2,
'S': 3
} # B:开始， I：中间词汇， S：单个字的词
self.class_size = len(self.ner_index_class)
self.classids = {}
for key in self.ner_index_class.keys():
self.classids[self.ner_index_class[key]] = key
self.graph = tf.Graph()
with self.graph.as_default():
with tf.variable_scope('ner_query'):
self.model = FenciNerCore(
io_sequence_size=self.io_sequence_size,
vocab_size=self.vocab_size,
class_size=self.class_size,
keep_prob=self.keep_prob,
trainable=self.is_training)
self.saver = tf.train.Saver()
config = tf.ConfigProto()
self.session = tf.Session(graph=self.graph, config=config)
with self.session.as_default():
self.load()
def load(self):
ckpt = tf.train.get_checkpoint_state(self.model_dir)
if ckpt is not None and ckpt.model_checkpoint_path:
self.saver.restore(self.session, ckpt.model_checkpoint_path)
print("ner load {} success...".format(
str(ckpt.model_checkpoint_path)))
def convert_row(self, input_text):
char_vector = np.zeros((self.io_sequence_size), dtype=np.int32)
for i in range(len(input_text[:self.io_sequence_size])):
# for i in range(len(input_text)):
char_value = input_text[i]
if char_value in self.char_index.keys():
char_vector[i] = self.char_index[char_value]
return char_vector
def predict(self, input_text):
input_text = input_text.strip().lower()
char_vector = self.convert_row(input_text.strip().lower())
seq_len_list = np.array([len(input_text)], dtype=np.int32)
feed_dict = {
self.model.inputs: np.array([char_vector], dtype=np.float32),
self.model.sequence_lengths: seq_len_list
}
logits, transition_params = self.session.run(
[self.model.logits, self.model.transition_params], feed_dict)
label_list = []
for logit, seq_len in zip(logits, seq_len_list):
viterbi_seq, _ = tf.contrib.crf.viterbi_decode(
logit[:seq_len], transition_params)
label_list.append(viterbi_seq)
if label_list is not None and len(label_list) > 0:
# taggs = []
# for i in range(seq_len_list[0]):
# taggs.append(self.classids[label_list[0][i]])
# output_labels = self.model.decode(list(input_text), taggs)
# data_items = {}
# for key in output_labels.keys():
# value = output_labels[key]
# #去除下标偏移
# data_items[key] = [vitem [0] for vitem in value]
# return data_items
return label_list
else:
return None
def to_char(self, label_list):
""""""
res = []
for i in label_list:
res.append([self.classids[w] for w in i])
return res
def test(self):
""""""
sentences, labels = self.data.get_test()
res = []
for i in range(len(sentences)):
# for i in range(100):
p = self.predict(sentences[i])
p = self.to_char(p)
res.append(int(labels[i] == ''.join(p[0])))
print(sum(res) / len(sentences))
def test2(self):
sentences, labels = self.data.get_test()
res = []
sum = 0
t_res = 0
# for i in range(len(sentences)):
for i in range(3000):
p = self.predict(sentences[i])
p = self.to_char(p)
for j in range(len(p[0])):
sum += 1
if labels[i][j] == p[0][j]:
t_res += 1
print(t_res / sum)
class NerTrainner:
def __init__(self):
self.model_dir = "./models//cws_2"
self.data = Data()
self.char_index = {
k: v
for v, k in enumerate(self.data.get_dictionary())
}
self.unknow_char_id = len(self.char_index)
self.io_sequence_size = 130
vocab_size = len(self.char_index) + 1
self.classnames = {
'O': 0,
'B': 1,
'I': 2,
'S': 3
} # B:开始， I：中间词汇， S：单个字的词
class_size = len(self.classnames) # brand,price,position,item,unknow
keep_prob = 0.5
learning_rate = 0.0005
trainable = True
self.batch_size = 128
with tf.variable_scope('ner_query'):
self.model = FenciNerCore(self.io_sequence_size, vocab_size,
class_size, keep_prob, learning_rate,
trainable)
def train(self, epochs=30):
records, lables = self.data.get_train()
print(len(records))
batch_count = int(len(records) / self.batch_size)
print("prepare data success ...")
initer = tf.global_variables_initializer()
with tf.Session(config=tf.ConfigProto(
gpu_options=gpu_options)) as session:
session.run(initer)
ckpt = tf.train.get_checkpoint_state(self.model_dir)
saver = tf.train.Saver()
if ckpt is not None and ckpt.model_checkpoint_path:
saver.restore(session, ckpt.model_checkpoint_path)
for epoch in range(epochs):
train_loss_value = 0.
# train_accuracy_value = 0.
for i in range(batch_count):
batch_records = records[i * self.batch_size:(i + 1) *
self.batch_size]
batch_labels = lables[i * self.batch_size:(i + 1) *
self.batch_size]
xrows, xlens, yrows = self.convert_batch(
batch_records, batch_labels)
feed_dict = {
self.model.inputs: xrows,
self.model.targets: yrows,
self.model.sequence_lengths: xlens
}
batch_loss_value, _ = session.run(
[self.model.cost_func, self.model.optimizer],
feed_dict)
train_loss_value += batch_loss_value / batch_count
if i % 100 == 0:
batch_buffer = "Progress {0}/{1} , cost : {2}".format(
i + 1, batch_count, batch_loss_value)
# print(batch_buffer, end="\r", flush=True)
print(batch_buffer)
print("Epoch: %d/%d , train cost=%f " %
((epoch + 1), epochs, train_loss_value))
saver.save(session, os.path.join(self.model_dir, "ner.dat"))
def convert_batch(self, sentense_list, label_list):
xrows = np.zeros((self.batch_size, self.io_sequence_size),
dtype=np.float32)
xlens = np.zeros((self.batch_size), dtype=np.int32)
yrows = np.zeros((self.batch_size, self.io_sequence_size),
dtype=np.int32)
count = len(sentense_list)
for i in range(count):
sent_text, tags = sentense_list[i][:-1], label_list[i][:-1]
xlen = len(sent_text)
if xlen > self.io_sequence_size:
# print(xlen)
xlen = self.io_sequence_size
xlens[i] = xlen
xrows[i] = self.convert_xrow(sent_text)
yrows[i] = self.convert_classids(tags)
return xrows, xlens, yrows
def convert_classids(self, tags):
yrow = np.zeros(self.io_sequence_size, dtype=np.int32)
for i in range(len(tags[:self.io_sequence_size])):
yrow[i] = self.classnames[tags[i]]
return yrow
def convert_xrow(self, input_text):
char_vector = np.zeros((self.io_sequence_size), dtype=np.int32)
for i in range(len(input_text[:self.io_sequence_size])):
char_value = input_text[i]
if char_value in self.char_index.keys():
char_vector[i] = self.char_index[char_value]
return char_vector