def train(self, sess, save_file, X_train_source, y_train_source, X_val_source, y_val_source,
X_train_target, y_train_target, X_val_target, y_val_target, X_test_source, y_test_source):
char2id, id2char = helper.loadMap("char2id")
label2id, id2label = helper.loadMap("label2id")
merged = tf.contrib.deprecated.merge_all_summaries()
summary_writer_train_source = tf.summary.FileWriter('loss_log/train_loss_source', sess.graph)
summary_writer_train_target = tf.summary.FileWriter('loss_log/train_loss_target', sess.graph)
summary_writer_val_source = tf.summary.FileWriter('loss_log/val_loss_source', sess.graph)
summary_writer_val_target = tf.summary.FileWriter('loss_log/val_loss_target', sess.graph)
len_source = len(X_train_source)
len_target = len(X_train_target)
# source_probability = len_source * 1.0 / (len_source + len_target)
source_probability = 0.65
count = 0
for epoch in range(self.num_epochs):
print "current epoch: %d" % (epoch)
if random.random() <= source_probability:
# sh_index = np.arange(len_source)
# np.random.shuffle(sh_index)
# X_train = X_train_source[sh_index]
# y_train = y_train_source[sh_index]
X_train = X_train_source
y_train = y_train_source
self.train_an_iteration(sess, save_file, X_train, y_train, X_val_source, y_val_source, char2id, id2char, label2id, id2label, summary_writer_train_source, summary_writer_train_target, summary_writer_val_source, summary_writer_val_target, flag="source", is_summary=(count%15==0), is_validation=(count%10==0), X_test_source=X_test_source, y_test_source=y_test_source)
else :
# sh_index = np.arange(len_target)
# np.random.shuffle(sh_index)
# X_train = X_train_target[sh_index]
# y_train = y_train_target[sh_index]
X_train = X_train_target
y_train = y_train_target
self.train_an_iteration(sess, save_file, X_train, y_train, X_val_target, y_val_target, char2id, id2char, label2id, id2label, summary_writer_train_source, summary_writer_train_target, summary_writer_val_source, summary_writer_val_target, flag="target", is_summary=(count%15==0), is_validation=(count%10==0), X_test_source=X_test_source, y_test_source=y_test_source)
count += 1
def test(self, sess, X_test, X_test_str, output_path):
char2id, id2char = helper.loadMap("char2id")
label2id, id2label = helper.loadMap("label2id")
num_iterations = int(math.ceil(1.0 * len(X_test) / self.batch_size))
print "number of iteration: " + str(num_iterations)
with open(output_path, "wb") as outfile:
for i in range(num_iterations):
print "iteration: " + str(i + 1)
results = []
X_test_batch = X_test[i * self.batch_size : (i + 1) * self.batch_size]
X_test_str_batch = X_test_str[i * self.batch_size : (i + 1) * self.batch_size]
if i == num_iterations - 1 and len(X_test_batch) < self.batch_size:
X_test_batch = list(X_test_batch)
X_test_str_batch = list(X_test_str_batch)
last_size = len(X_test_batch)
X_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_test_str_batch += [['x' for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_test_batch = np.array(X_test_batch)
X_test_str_batch = np.array(X_test_str_batch)
results = self.predictBatch(sess, X_test_batch, X_test_str_batch, id2label)
results = results[:last_size]
else:
X_test_batch = np.array(X_test_batch)
results = self.predictBatch(sess, X_test_batch, X_test_str_batch, id2label)
for i in range(len(results)):
doc = ''.join(X_test_str_batch[i])
outfile.write(doc + "<@>" +" ".join(results[i]).encode("utf-8") + "\n")
def predict(self, sess, X_predict, X_predict_str, output_path):
char2id, id2char = helper.loadMap("char2id")
label2id, id2label = helper.loadMap("label2id")
num_iterations = int(math.ceil(1.0 * len(X_predict) / self.batch_size))
print("number of iteration: " + str(num_iterations))
with open(output_path + '.ann', "w") as outfile:
for i in range(num_iterations):
print("predict iteration: " + str(i + 1))
results = []
X_predict_batch = X_predict[i * self.batch_size:(i + 1) *
self.batch_size]
X_predict_str_batch = X_predict_str[i *
self.batch_size:(i + 1) *
self.batch_size]
if i == num_iterations - 1 and len(
X_predict_batch) < self.batch_size:
X_predict_batch = list(X_predict_batch)
X_predict_str_batch = list(X_predict_str_batch)
last_size = len(X_predict_batch)
X_predict_batch += [[0 for j in range(self.num_steps)]
for i in range(self.batch_size -
last_size)]
X_predict_str_batch += [[
'None' for j in range(self.num_steps)
] for i in range(self.batch_size - last_size)]
X_predict_batch = np.array(X_predict_batch)
X_predict_str_batch = np.array(X_predict_str_batch)
results = self.predictBatch(sess, X_predict_batch,
X_predict_str_batch, id2label)
results = results[:last_size]
else:
X_predict_batch = np.array(X_predict_batch)
predicts, results = self.predictBatch(
sess, X_predict_batch, X_predict_str_batch, id2label)
results = results[:len(X_predict_batch)]
rawtext = ''
count = 0
for j in range(len(X_predict_str_batch)):
try:
for word, label in zip(X_predict_str_batch[j],
results[1][j]):
count += 1
merge_line = ''
merge_line = 'T' + str(count) + '\t' + str(
label) + ' ' + str(len(rawtext)) + ' ' + str(
len(rawtext) +
len(str(word))) + '\t' + str(word) + '\n'
rawtext += str(word) + ' '
if str(label) != 'others':
outfile.write(merge_line)
print(merge_line)
except Exception:
continue
outfile.close()
with open(output_path + '.txt', 'w') as f:
f.write(rawtext.strip())
f.close()
def test(self, sess, X_test, X_test_str, output_path, y_test=None):
char2id, id2char = helper.loadMap("char2id")
label2id, id2label = helper.loadMap("label2id")
precision_mean = 0
recall_mean = 0
y_predicts = []
num_iterations = int(math.ceil(1.0 * len(X_test) / self.batch_size))
print("number of iteration: " + str(num_iterations))
with open(output_path, "wb") as outfile:
for i in range(num_iterations):
print("Test iteration: " + str(i + 1))
results = []
X_test_batch = X_test[i * self.batch_size:(i + 1) *
self.batch_size]
X_test_str_batch = X_test_str[i * self.batch_size:(i + 1) *
self.batch_size]
y_test_batch = y_test[i * self.batch_size:(i + 1) *
self.batch_size]
if i == num_iterations - 1 and len(
X_test_batch) < self.batch_size:
X_test_batch = list(X_test_batch)
X_test_str_batch = list(X_test_str_batch)
last_size = len(X_test_batch)
X_test_batch += [[0 for j in range(self.num_steps)]
for i in range(self.batch_size -
last_size)]
X_test_str_batch += [['x' for j in range(self.num_steps)]
for i in range(self.batch_size -
last_size)]
X_test_batch = np.array(X_test_batch)
X_test_str_batch = np.array(X_test_str_batch)
predicts, results = self.predictBatch(
sess, X_test_batch, X_test_str_batch, id2label)
results = results[:last_size]
else:
X_test_batch = np.array(X_test_batch)
predicts, results = self.predictBatch(
sess, X_test_batch, X_test_str_batch, id2label)
results = results[:len(X_test_batch)]
predicts = predicts[:len(X_test_batch)] #最後一批
y_predicts += predicts
# precision_val, recall_val, f1_val = self.evaluate_test( y_test_batch, results, id2label)
pre_num, true_num, hit_num, precision_test, recall_test, f1_test = self.evaluate(
X_test, y_test, y_predicts, id2char, id2label)
print(
"predicted num:%d, true num:%d, hit num:%d, test precision: %.5f, test recall: %.5f, test f1: %.5f"
% (pre_num, true_num, hit_num, precision_test, recall_test,
f1_test))
def train(self,train_data):
# TODO 验证集
X_train = train_data['char']
X_left_train = train_data['left']
X_right_train = train_data['right']
X_pos_train = train_data['pos']
X_lpos_train = train_data['lpos']
X_rpos_train = train_data['rpos']
X_rel_train = train_data['rel']
X_dis_train = train_data['dis']
y_train = train_data['label']
char2id, id2char = helper.loadMap("char2id")
pos2id, id2pos = helper.loadMap("pos2id")
label2id, id2label = helper.loadMap("label2id")
num_iterations = int(math.ceil(1.0 * len(X_train) / self.batch_size)) #每轮次数
'''
dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(dir)
num_epochs = 10 #args.epoch
emb_path = os.path.join(dir, 'embedding/embedding.npy') #args.char_emb
gpu_config = "/gpu:0" #+str(args.gpu)
model_path = os.path.join(dir, 'model', 'NER') #args.model_path
predict_path = os.path.join(dir, 'data', 'candidate_predict')
predict_output_path = os.path.join(dir, 'data', 'rawdata',
'test.txt') #args.output_path
num_steps = 200 #it must consist with the train
start_time = time.time()
char2id, id2char = helper.loadMap("char2id")
label2id, id2label = helper.loadMap("label2id")
num_chars = len(id2char.keys())
num_classes = len(id2label.keys())
if emb_path != None:
embedding_matrix = helper.getEmbedding(emb_path)
else:
embedding_matrix = None
print("building model")
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
with tf.device(gpu_config):
initializer = tf.random_uniform_initializer(-0.1, 0.1)
with tf.variable_scope("model", reuse=None, initializer=initializer):
model = BILSTM_CRF(num_chars=num_chars,
def train(self, sess, save_file, X_train, y_train, X_val, y_val):
saver = tf.train.Saver()
char2id, id2char = helper.loadMap("char2id")
label2id, id2label = helper.loadMap("label2id")
merged = tf.contrib.deprecated.merge_all_summaries()
summary_writer_train = tf.contrib.summary.SummaryWriter(
'loss_log/train_loss', sess.graph)
summary_writer_val = tf.contrib.summary.SummaryWriter(
'loss_log/val_loss', sess.graph)
num_iterations = int(math.ceil(1.0 * len(X_train) / self.batch_size))
cnt = 0
for epoch in range(self.num_epochs):
# shuffle train in each epoch
sh_index = np.arange(len(X_train))
np.random.shuffle(sh_index)
X_train = X_train[sh_index]
y_train = y_train[sh_index]
print("current epoch: %d" % (epoch))
for iteration in range(num_iterations):
# train
X_train_batch, y_train_batch = helper.nextBatch(
X_train,
y_train,
start_index=iteration * self.batch_size,
batch_size=self.batch_size)
y_train_weight_batch = 1 + np.array(
(y_train_batch == label2id['B']) |
(y_train_batch == label2id['E']), float)
transition_batch = helper.getTransition(y_train_batch)
_, loss_train, max_scores, max_scores_pre, length, train_summary = \
sess.run([
self.optimizer,
self.loss,
self.max_scores,
self.max_scores_pre,
self.length,
self.train_summary
],
feed_dict={
self.targets_transition: transition_batch,
self.inputs: X_train_batch,
self.targets: y_train_batch,
self.targets_weight: y_train_weight_batch
})
predicts_train = self.viterbi(max_scores,
max_scores_pre,
length,
predict_size=self.batch_size)
if iteration % 10 == 0:
cnt += 1
precision_train, recall_train, f1_train = self.evaluate(
X_train_batch, y_train_batch, predicts_train, id2char,
id2label)
summary_writer_train.add_summary(train_summary, cnt)
print(
"iteration: %5d, train loss: %5d, train precision: %.5f, train recall: %.5f, train f1: %.5f"
% (iteration, loss_train, precision_train,
recall_train, f1_train))
# validation
if iteration % 100 == 0:
X_val_batch, y_val_batch = helper.nextRandomBatch(
X_val, y_val, batch_size=self.batch_size)
y_val_weight_batch = 1 + np.array(
(y_val_batch == label2id['B']) |
(y_val_batch == label2id['E']), float)
transition_batch = helper.getTransition(y_val_batch)
loss_val, max_scores, max_scores_pre, length, val_summary = \
sess.run([
self.loss,
self.max_scores,
self.max_scores_pre,
self.length,
self.val_summary
],
feed_dict={
self.targets_transition: transition_batch,
self.inputs: X_val_batch,
self.targets: y_val_batch,
self.targets_weight: y_val_weight_batch
})
predicts_val = self.viterbi(max_scores,
max_scores_pre,
length,
predict_size=self.batch_size)
precision_val, recall_val, f1_val = self.evaluate(
X_val_batch, y_val_batch, predicts_val, id2char,
id2label)
summary_writer_val.add_summary(val_summary, cnt)
print(
"iteration: %5d, valid loss: %5d, valid precision: %.5f, valid recall: %.5f, valid f1: %.5f"
% (iteration, loss_val, precision_val, recall_val,
f1_val))
if f1_val > self.max_f1:
self.max_f1 = f1_val
save_path = saver.save(sess, save_file)
print("saved the best model with f1: %.5f" %
(self.max_f1))
def test(self, sess, test_data, output_path):
#data
X_test = test_data['char']
X_left_test = test_data['left']
X_right_test = test_data['right']
X_pos_test = test_data['pos']
X_lpos_test = test_data['lpos']
X_rpos_test = test_data['rpos']
X_rel_test = test_data['rel']
X_dis_test = test_data['dis']
#dictionary
char2id, id2char = helper.loadMap("char2id")
pos2id, id2pos = helper.loadMap("pos2id")
label2id, id2label = helper.loadMap("label2id")
num_iterations = int(math.ceil(1.0 * len(X_test) / self.batch_size))
print "number of iteration: " + str(num_iterations)
with open(output_path, "wb") as outfile:
pred_lines = []
for i in range(num_iterations):
print "iteration: " + str(i + 1)
results = []
#get batch
X_test_batch = X_test[i * self.batch_size : (i + 1) * self.batch_size]
X_left_test_batch = X_left_test[i * self.batch_size : (i + 1) * self.batch_size]
X_right_test_batch = X_right_test[i * self.batch_size : (i + 1) * self.batch_size]
X_pos_test_batch = X_pos_test[i * self.batch_size : (i + 1) * self.batch_size]
X_lpos_test_batch = X_lpos_test[i * self.batch_size : (i + 1) * self.batch_size]
X_rpos_test_batch = X_rpos_test[i * self.batch_size : (i + 1) * self.batch_size]
X_rel_test_batch = X_rel_test[i * self.batch_size : (i + 1) * self.batch_size]
X_dis_test_batch = X_dis_test[i * self.batch_size : (i + 1) * self.batch_size]
# left seqtence less than batch size, use [0] as seq
if i == num_iterations - 1 and len(X_test_batch) < self.batch_size:
X_test_batch = list(X_test_batch)
X_left_test_batch = list(X_left_test_batch)
X_right_test_batch = list(X_right_test_batch)
X_pos_test_batch = list(X_pos_test_batch)
X_lpos_test_batch = list(X_lpos_test_batch)
X_rpos_test_batch = list(X_rpos_test_batch)
X_rel_test_batch = list(X_rel_test_batch)
X_dis_test_batch = list(X_dis_test_batch)
last_size = len(X_test_batch)
X_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_left_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_right_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_pos_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_lpos_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_rpos_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_rel_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_dis_test_batch += [[0 for j in range(self.num_steps)] for i in range(self.batch_size - last_size)]
X_test_batch = np.array(X_test_batch)
X_left_test_batch = np.array(X_left_test_batch)
X_right_test_batch = np.array(X_right_test_batch)
X_pos_test_batch = np.array(X_pos_test_batch)
X_lpos_test_batch = np.array(X_lpos_test_batch)
X_rpos_test_batch = np.array(X_rpos_test_batch)
X_rel_test_batch = np.array(X_rel_test_batch)
X_dis_test_batch = np.array(X_dis_test_batch)
test_batches = {}
test_batches['char'] = X_test_batch
test_batches['left'] = X_left_test_batch
test_batches['right'] = X_right_test_batch
test_batches['pos'] = X_pos_test_batch
test_batches['lpos'] = X_lpos_test_batch
test_batches['rpos'] = X_rpos_test_batch
test_batches['rel'] = X_rel_test_batch
test_batches['dis'] = X_dis_test_batch
results = self.predictBatch(sess, test_batches, id2label)
results = results[:last_size]
pred_lines.extend(results)
else: # next batch
X_test_batch = np.array(X_test_batch)
X_left_test_batch = np.array(X_left_test_batch)
X_right_test_batch = np.array(X_right_test_batch)
X_pos_test_batch = np.array(X_pos_test_batch)
X_lpos_test_batch = np.array(X_lpos_test_batch)
X_rpos_test_batch = np.array(X_rpos_test_batch)
X_rel_test_batch = np.array(X_rel_test_batch)
X_dis_test_batch = np.array(X_dis_test_batch)
test_batches = {}
test_batches['char'] = X_test_batch
test_batches['left'] = X_left_test_batch
test_batches['right'] = X_right_test_batch
test_batches['pos'] = X_pos_test_batch
test_batches['lpos'] = X_lpos_test_batch
test_batches['rpos'] = X_rpos_test_batch
test_batches['rel'] = X_rel_test_batch
test_batches['dis'] = X_dis_test_batch
results = self.predictBatch(sess, test_batches, id2label)
pred_lines.extend(results)
return pred_lines
def train(self, sess, save_file, train_data, val_data):
saver = tf.train.Saver(max_to_keep=3)
#train data
X_train = train_data['char']
X_left_train = train_data['left']
X_right_train = train_data['right']
X_pos_train = train_data['pos']
X_lpos_train = train_data['lpos']
X_rpos_train = train_data['rpos']
X_rel_train = train_data['rel']
X_dis_train = train_data['dis']
y_train = train_data['label']
#dev data
X_val = val_data['char']
X_left_val = val_data['left']
X_right_val = val_data['right']
X_pos_val = val_data['pos']
X_lpos_val = val_data['lpos']
X_rpos_val = val_data['rpos']
X_rel_val = val_data['rel']
X_dis_val = val_data['dis']
y_val = val_data['label']
#dictionary
char2id, id2char = helper.loadMap("char2id")
pos2id, id2pos = helper.loadMap("pos2id")
label2id, id2label = helper.loadMap("label2id")
merged = tf.summary.merge_all()
summary_writer_train = tf.summary.FileWriter('loss_log/train_loss', sess.graph)
summary_writer_val = tf.summary.FileWriter('loss_log/val_loss', sess.graph)
num_iterations = int(math.ceil(1.0 * len(X_train) / self.batch_size))
cnt = 0
for epoch in range(self.num_epochs):
# shuffle train in each epoch
sh_index = np.arange(len(X_train))
np.random.shuffle(sh_index)
X_train = X_train[sh_index]
X_left_train = X_left_train[sh_index]
X_right_train = X_right_train[sh_index]
X_pos_train = X_pos_train[sh_index]
X_lpos_train = X_lpos_train[sh_index]
X_rpos_train = X_rpos_train[sh_index]
X_rel_train = X_rel_train[sh_index]
X_dis_train = X_dis_train[sh_index]
y_train = y_train[sh_index]
train_data['char'] = X_train
train_data['left'] = X_left_train
train_data['right'] = X_right_train
train_data['pos'] = X_pos_train
train_data['lpos'] = X_lpos_train
train_data['rpos'] = X_rpos_train
train_data['rel'] = X_rel_train
train_data['dis'] = X_dis_train
train_data['label'] = y_train
print "current epoch: %d" % (epoch)
for iteration in range(num_iterations):
# train
#get batch
train_batches = helper.nextBatch(train_data, start_index=iteration * self.batch_size, batch_size=self.batch_size)
X_train_batch = train_batches['char']
X_left_train_batch = train_batches['left']
X_right_train_batch = train_batches['right']
X_pos_train_batch = train_batches['pos']
X_lpos_train_batch = train_batches['lpos']
X_rpos_train_batch = train_batches['rpos']
X_rel_train_batch = train_batches['rel']
X_dis_train_batch = train_batches['dis']
y_train_batch = train_batches['label']
# feed batch to model and run
_, loss_train, length, train_summary, logits, trans_params =\
sess.run([
self.optimizer,
self.loss,
self.length,
self.train_summary,
self.logits,
self.trans_params,
],
feed_dict={
self.inputs:X_train_batch,
self.lefts:X_left_train_batch,
self.rights:X_right_train_batch,
self.poses:X_pos_train_batch,
self.lposes:X_lpos_train_batch,
self.rposes:X_rpos_train_batch,
self.rels:X_rel_train_batch,
self.dises:X_dis_train_batch,
self.targets:y_train_batch
# self.targets_weight:y_train_weight_batch
})
# print (len(length))
#get predict f1
predicts_train = self.viterbi(logits, trans_params, length, predict_size=self.batch_size)
if iteration > 0 and iteration % 10 == 0:
cnt += 1
hit_num, pred_num, true_num = self.evaluate(y_train_batch, predicts_train, id2char, id2label)
precision_train, recall_train, f1_train = self.caculate(hit_num, pred_num, true_num)
summary_writer_train.add_summary(train_summary, cnt)
print "iteration: %5d/%5d, train loss: %5d, train precision: %.5f, train recall: %.5f, train f1: %.5f" % (iteration, num_iterations, loss_train, precision_train, recall_train, f1_train)
# a batch in validation
if iteration > 0 and iteration % 100 == 0:
val_batches = helper.nextRandomBatch(val_data, batch_size=self.batch_size)
X_val_batch = val_batches['char']
X_left_val_batch = val_batches['left']
X_right_val_batch = val_batches['right']
X_pos_val_batch = val_batches['pos']
X_lpos_val_batch = val_batches['lpos']
X_rpos_val_batch = val_batches['rpos']
X_rel_val_batch = val_batches['rel']
X_dis_val_batch = val_batches['dis']
y_val_batch = val_batches['label']
loss_val, length, val_summary, logits, trans_params =\
sess.run([
self.loss,
self.length,
self.val_summary,
self.logits,
self.trans_params,
],
feed_dict={
self.inputs:X_val_batch,
self.lefts:X_left_val_batch,
self.rights:X_right_val_batch,
self.poses:X_pos_val_batch,
self.lposes:X_lpos_val_batch,
self.rposes:X_rpos_val_batch,
self.rels:X_rel_val_batch,
self.dises:X_dis_val_batch,
self.targets:y_val_batch
# self.targets_weight:y_val_weight_batch
})
predicts_val = self.viterbi(logits, trans_params, length, predict_size=self.batch_size)
hit_num, pred_num, true_num = self.evaluate(y_val_batch, predicts_val, id2char, id2label)
precision_val, recall_val, f1_val = self.caculate(hit_num, pred_num, true_num)
summary_writer_val.add_summary(val_summary, cnt)
print "iteration: %5d, valid loss: %5d, valid precision: %.5f, valid recall: %.5f, valid f1: %.5f" % (iteration, loss_val, precision_val, recall_val, f1_val)
# calc f1 for the whole dev set
if epoch > 0 and iteration == num_iterations -1:
num_val_iterations = int(math.ceil(1.0 * len(X_val) / self.batch_size))
preds_lines = []
for val_iteration in range(num_val_iterations):
val_batches = helper.nextBatch(val_data, start_index=val_iteration * self.batch_size, batch_size=self.batch_size)
X_val_batch = val_batches['char']
X_left_val_batch = val_batches['left']
X_right_val_batch = val_batches['right']
X_pos_val_batch = val_batches['pos']
X_lpos_val_batch = val_batches['lpos']
X_rpos_val_batch = val_batches['rpos']
X_rel_val_batch = val_batches['rel']
X_dis_val_batch = val_batches['dis']
y_val_batch = val_batches['label']
loss_val, length, val_summary, logits, trans_params =\
sess.run([
self.loss,
self.length,
self.val_summary,
self.logits,
self.trans_params,
],
feed_dict={
self.inputs:X_val_batch,
self.lefts:X_left_val_batch,
self.rights:X_right_val_batch,
self.poses:X_pos_val_batch,
self.lposes:X_lpos_val_batch,
self.rposes:X_rpos_val_batch,
self.rels:X_rel_val_batch,
self.dises:X_dis_val_batch,
self.targets:y_val_batch
# self.targets_weight:y_val_weight_batch
})
predicts_val = self.viterbi(logits, trans_params, length, predict_size=self.batch_size)
preds_lines.extend(predicts_val)
preds_lines = preds_lines[:len(y_val)]
recall_val, precision_val, f1_val, errors = helper.calc_f1(preds_lines, id2label, 'cpbdev.txt', 'validation.out')
if f1_val > self.max_f1:
self.max_f1 = f1_val
save_path = saver.save(sess, save_file)
helper.calc_f1(preds_lines, id2label, 'cpbdev.txt', 'validation.out.best')
print "saved the best model with f1: %.5f" % (self.max_f1)
print "valid precision: %.5f, valid recall: %.5f, valid f1: %.5f, errors: %5d" % (precision_val, recall_val, f1_val, errors)
def main(_):
#initial outer file
WordIndex = Word_Index(FLAGS.word2id_path)
TarIndex = Tar_Tag_Index()
OpiIndex = Opi_Tag_Index()
FLAGS.num_word = len(WordIndex.word2idex)
FLAGS.num_class = TarIndex.num_class
df_test = pd.read_csv(FLAGS.test_file, sep='#', skip_blank_lines=False, dtype={'len': np.int32})
df_train = pd.read_csv(FLAGS.train_file, sep='#', skip_blank_lines=False, dtype={'len': np.int32})
df_train = df_train.iloc[np.random.permutation(len(df_train))].reset_index()
eval_size = int(len(df_train) * FLAGS.dev_rate)
df_eval = df_train.iloc[-eval_size:]
df_train = df_train.iloc[:-eval_size]
print('trainsize' + str(len(df_train)))
train_data_itor = DataItor(df_train)
eval_data_itor = DataItor(df_eval)
test_data_itor = DataItor(df_test)
FLAGS.check_every_point = int(train_data_itor.size / FLAGS.batch_size)
word2id, id2word = helper.loadMap(FLAGS.word2id_path)
if os.path.exists(FLAGS.pretrain_file):
FLAGS.pretrain_emb = initial_embedding_yelp_bin(word2id)
else:
FLAGS.pretrain_emb = None
myconfig = tf.ConfigProto(allow_soft_placement = True)
with tf.Session(config=myconfig) as sess:
model = BILSTM(FLAGS)
sess.run(tf.global_variables_initializer())
w_xs_eval, y_tuple_eval, couple_eval, lens_eval = eval_data_itor.next_all(FLAGS.use_couple)
w_xs_test, y_tuple_test, couple_test, lens_test = test_data_itor.next_all_no_padding(FLAGS.use_couple)
print('eval data size %f' % len(w_xs_eval))
# _, id2label = helper.loadMap(FLAGS.label2id_path)
saver = tf.train.Saver(max_to_keep=2)
previous_best_valid_f1_score = 0
previous_best_epoch = -1
bad_count = 0
heap_target, heap_opword = [], []
while train_data_itor.epoch < FLAGS.num_epochs:
x_train_batch, y_train_batch,couple_train_batch = train_data_itor.next_batch(FLAGS.batch_size,FLAGS.use_couple)
train_step, train_loss = model.train_model_union(sess, x_train_batch, y_train_batch,couples=couple_train_batch)
if train_data_itor.batch_time % FLAGS.check_every_point == 0:
print("current batch_time: %d" % (train_data_itor.batch_time))
opword_y_eval_pred, target_y_eval_pred, eval_loss, _,_ = model.inference_for_cpu(sess, w_xs_eval, y_tuple_eval, couple_eval)
# print('every loss:%f,%f' % (ent_loss, opi_loss))
precison, recall, target_f1_eval = helper.evaluate(w_xs_eval, y_tuple_eval[0],
target_y_eval_pred, id2word=id2word,seq_lens=lens_eval,
label_type='target')
print('evalution on eval data, target_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f' % ( eval_loss, precison, recall, target_f1_eval))
opword_y_test_pred, target_y_test_pred, test_loss, all_ent_att_scores, all_opi_att_scores = model.inference_for_cpu(sess, w_xs_test, y_tuple_test,couple_test)
precison1, recall1, target_f1_test = helper.evaluate(w_xs_test, y_tuple_test[0],
target_y_test_pred, id2word=id2word,seq_lens=lens_test,
label_type='target')
print('evalution on test data, target_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f' % ( test_loss, precison1, recall1, target_f1_test))
opword_precison, opword_recall, opword_f1_eval = helper.evaluate(w_xs_eval, y_tuple_eval[1],
opword_y_eval_pred, id2word=id2word,seq_lens=lens_eval,
label_type='opword')
print('evalution on eval data, opword_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f' % (eval_loss, opword_precison, opword_recall, opword_f1_eval))
# opword_y_test_pred, opword_test_loss = model.decode_opinion(sess, test_datas, y_opinions_test)
opword_precison, opword_recall, opword_f1_test = helper.evaluate(w_xs_test, y_tuple_test[1], opword_y_test_pred,
id2word=id2word,seq_lens=lens_test,
label_type='opword')
print('evalution on test data, opword_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f' % (test_loss, opword_precison, opword_recall, opword_f1_test))
if len(heap_target) < 5:
heapq.heappush(heap_target, (target_f1_eval, train_data_itor.epoch, target_f1_test,opword_f1_test))
else:
if target_f1_eval > heap_target[0][0]:
_, delete_file_epoch,_,_ = heapq.heappop(heap_target)
heapq.heappush(heap_target, (target_f1_eval, train_data_itor.epoch, target_f1_test,opword_f1_test))
if len(heap_opword) < 5:
heapq.heappush(heap_opword,(opword_f1_eval, train_data_itor.epoch, opword_f1_test))
else:
if opword_f1_eval > heap_opword[0][0]:
_, delete_file_epoch, _ = heapq.heappop(heap_opword)
heapq.heappush(heap_opword, (opword_f1_eval, train_data_itor.epoch, opword_f1_test))
# early stop
if target_f1_eval > previous_best_valid_f1_score:
previous_best_valid_f1_score = target_f1_eval
bad_count = 0
store_weights(all_ent_att_scores)
else:
bad_count += 1
if bad_count >= FLAGS.patients:
print('early stop!')
break
print('Train Finished!!')
# writer = codecs.open(exp_paths[task_id], 'a', 'utf-8')
# writer.close()
show_result(heap_target)
# show_result(heap_opword)
pass
from sklearn import model_selection
from sklearn.externals import joblib
import tensorflow as tf
import time
import helper
import sys, os
import numpy as np
dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(dir)
train_path = os.path.join(dir, 'rawdatarela_train.txt')
embedding_path = os.path.join(dir, 'embedding.npy')
save_path = os.path.join(dir, 'model')
start_time = time.time()
word2id, id2word = helper.loadMap("word2id.txt")
label2id, id2label = helper.loadMap("label2id.txt")
entitylabel2id, id2entitylabel = helper.loadMap("entitylabel2id.txt")
num_words = len(id2word.keys())
num_classes = len(id2label.keys())
emb_dim = 128
batch_size = 128
print("preparing train and validation data")
label, entity1label, entity2label, entity1, entity2, distance = helper.getTrainData(
train_path=train_path)
if embedding_path != None:
embedding_matrix = helper.getEmbedding(embedding_path, emb_dim=emb_dim)
else:
embedding_matrix = None
features = np.transpose(np.array([entity1label, entity2label, distance]))
input_entity1_emb = np.zeros((len(entity1), emb_dim))
args = parser.parse_args()
train_path = args.train_path
save_path = args.save_path
val_path = args.val_path
num_epochs = args.epoch
emb_path = args.char_emb
# gpu_config = "/gpu:"+str(args.gpu)
gpu_config = "/cpu:0"
num_steps = 200 # it must consist with the test
start_time = time.time()
print "preparing train and validation data"
X_train, y_train, X_val, y_val = helper.getTrain(train_path=train_path, val_path=val_path, seq_max_len=num_steps)
char2id, id2char = helper.loadMap("char2id")
label2id, id2label = helper.loadMap("label2id")
num_chars = len(id2char.keys())
num_classes = len(id2label.keys())
if emb_path != None:
embedding_matrix = helper.getEmbedding(emb_path)
else:
embedding_matrix = None
print "building model"
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
with tf.device(gpu_config):
initializer = tf.random_uniform_initializer(-0.1, 0.1)
with tf.variable_scope("model", reuse=None, initializer=initializer):
model = BILSTM_CRF(num_chars=num_chars, num_classes=num_classes, num_steps=num_steps, num_epochs=num_epochs, embedding_matrix=embedding_matrix, is_training=True)
def train(self, sess, save_file, X_train, y_train, X_val, y_val):
saver = tf.train.Saver()
char2id, id2char = helper.loadMap("char2id")
label2id, id2label = helper.loadMap("label2id")
merged = tf.merge_all_summaries()
summary_writer_train = tf.train.SummaryWriter('loss_log/train_loss', sess.graph)
summary_writer_val = tf.train.SummaryWriter('loss_log/val_loss', sess.graph)
num_iterations = int(math.ceil(1.0 * len(X_train) / self.batch_size))
cnt = 0
for epoch in range(self.num_epochs):
# shuffle train in each epoch
sh_index = np.arange(len(X_train))
np.random.shuffle(sh_index)
X_train = X_train[sh_index]
y_train = y_train[sh_index]
print "current epoch: %d" % (epoch)
for iteration in range(num_iterations):
# train
X_train_batch, y_train_batch = helper.nextBatch(X_train, y_train, start_index=iteration * self.batch_size, batch_size=self.batch_size)
y_train_weight_batch = 1 + np.array((y_train_batch == label2id['B']) | (y_train_batch == label2id['E']), float)
transition_batch = helper.getTransition(y_train_batch)
_, loss_train, max_scores, max_scores_pre, length, train_summary =\
sess.run([
self.optimizer,
self.loss,
self.max_scores,
self.max_scores_pre,
self.length,
self.train_summary
],
feed_dict={
self.targets_transition:transition_batch,
self.inputs:X_train_batch,
self.targets:y_train_batch,
self.targets_weight:y_train_weight_batch
})
predicts_train = self.viterbi(max_scores, max_scores_pre, length, predict_size=self.batch_size)
if iteration % 10 == 0:
cnt += 1
precision_train, recall_train, f1_train = self.evaluate(X_train_batch, y_train_batch, predicts_train, id2char, id2label)
summary_writer_train.add_summary(train_summary, cnt)
print "iteration: %5d, train loss: %5d, train precision: %.5f, train recall: %.5f, train f1: %.5f" % (iteration, loss_train, precision_train, recall_train, f1_train)
# validation
if iteration % 100 == 0:
X_val_batch, y_val_batch = helper.nextRandomBatch(X_val, y_val, batch_size=self.batch_size)
y_val_weight_batch = 1 + np.array((y_val_batch == label2id['B']) | (y_val_batch == label2id['E']), float)
transition_batch = helper.getTransition(y_val_batch)
loss_val, max_scores, max_scores_pre, length, val_summary =\
sess.run([
self.loss,
self.max_scores,
self.max_scores_pre,
self.length,
self.val_summary
],
feed_dict={
self.targets_transition:transition_batch,
self.inputs:X_val_batch,
self.targets:y_val_batch,
self.targets_weight:y_val_weight_batch
})
predicts_val = self.viterbi(max_scores, max_scores_pre, length, predict_size=self.batch_size)
precision_val, recall_val, f1_val = self.evaluate(X_val_batch, y_val_batch, predicts_val, id2char, id2label)
summary_writer_val.add_summary(val_summary, cnt)
print "iteration: %5d, valid loss: %5d, valid precision: %.5f, valid recall: %.5f, valid f1: %.5f" % (iteration, loss_val, precision_val, recall_val, f1_val)
if f1_val > self.max_f1:
self.max_f1 = f1_val
save_path = saver.save(sess, save_file)
print "saved the best model with f1: %.5f" % (self.max_f1)
def main(_):
#initial outer file
WordIndex = Word_Index(FLAGS.word2id_path)
Index = [Tar_Tag_Index(), Opi_Tag_Index()]
object_tag_Index = Index[FLAGS.object_id]
FLAGS.dev_rate = 0.1
FLAGS.num_word = len(WordIndex.word2idex)
FLAGS.num_class = object_tag_Index.num_class
df_test = pd.read_csv(FLAGS.test_file,
sep='#',
skip_blank_lines=False,
dtype={'len': np.int32})
df_train = pd.read_csv(FLAGS.train_file,
sep='#',
skip_blank_lines=False,
dtype={'len': np.int32})
random_index = np.random.permutation(len(df_train))
df_train = df_train.iloc[random_index].reset_index()
print(random_index)
for i in range(len(random_index)):
print(random_index[i])
eval_size = int(len(df_train) * FLAGS.dev_rate)
df_eval = df_train.iloc[-eval_size:]
df_train = df_train.iloc[:-eval_size]
print('trainsize' + str(len(df_train)))
train_data_itor = DataItor(df_train)
eval_data_itor = DataItor(df_eval)
test_data_itor = DataItor(df_test)
FLAGS.check_every_point = int(train_data_itor.size / FLAGS.batch_size)
word2id, id2word = helper.loadMap(FLAGS.word2id_path)
if os.path.exists(FLAGS.pretrain_file):
FLAGS.pretrain_emb = initial_embedding_yelp_bin(word2id)
else:
FLAGS.pretrain_emb = None
myconfig = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=myconfig) as sess:
model = BILSTM(FLAGS)
sess.run(tf.global_variables_initializer())
w_xs_eval, y_tuple_eval, _, lens_eval = eval_data_itor.next_all(False)
w_xs_test, y_tuple_test, _, lens_test = test_data_itor.next_all_no_padding(
False)
print('eval data size %f' % len(w_xs_eval))
# _, id2label = helper.loadMap(FLAGS.label2id_path)
saver = tf.train.Saver(max_to_keep=2)
previous_best_valid_f1_score = 0
previous_best_epoch = -1
bad_count = 0
heap_target, heap_opword = [], []
while train_data_itor.epoch < FLAGS.num_epochs:
x_train_batch, y_train_batch, _ = train_data_itor.next_batch(
FLAGS.batch_size)
train_step, train_loss = model.train_model(
sess, x_train_batch, y_train_batch[FLAGS.object_id])
if train_data_itor.batch_time % FLAGS.check_every_point == 0:
print("current batch_time: %d" % (train_data_itor.batch_time))
y_eval_pred, eval_loss = model.inference_for_single(
sess, w_xs_eval, y_tuple_eval[FLAGS.object_id])
precison, recall, target_f1_eval = helper.evaluate(
w_xs_eval,
y_tuple_eval[FLAGS.object_id],
y_eval_pred,
id2word=id2word,
seq_lens=lens_eval,
label_type=types[FLAGS.object_id])
print(
'evalution on eval data, target_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f'
% (eval_loss, precison, recall, target_f1_eval))
y_test_pred, test_loss = model.inference_for_single(
sess, w_xs_test, y_tuple_test[FLAGS.object_id])
precison1, recall1, target_f1_test = helper.evaluate(
w_xs_test,
y_tuple_test[FLAGS.object_id],
y_test_pred,
id2word=id2word,
seq_lens=lens_test,
label_type=types[FLAGS.object_id])
print(
'evalution on test data, target_eval_loss:%.3f,precison:%.3f,recall:%.3f,fscore:%.3f'
% (test_loss, precison1, recall1, target_f1_test))
if len(heap_target) < 5:
heapq.heappush(heap_target,
(target_f1_eval, train_data_itor.epoch,
target_f1_test))
else:
if target_f1_eval > heap_target[0][0]:
_, delete_file_epoch, _ = heapq.heappop(heap_target)
heapq.heappush(heap_target,
(target_f1_eval, train_data_itor.epoch,
target_f1_test))
# early stop
if target_f1_eval > previous_best_valid_f1_score:
previous_best_valid_f1_score = target_f1_eval
bad_count = 0
else:
bad_count += 1
if bad_count >= FLAGS.patients:
print('early stop!')
break
print('Train Finished!!')
show_result(heap_target)
pass
def train(self, sess, saver, save_file, X_train, y_train, X_valid, y_valid,
X_train_tag, X_valid_tag, y_intent_train, y_intent_valid,
model_dev, seq_len_train, seq_len_valid):
char2id, id2char = helper.loadMap("meta_data/char2id")
label2id, id2label = helper.loadMap("meta_data/label2id")
num_iterations = int(math.ceil(1.0 * len(X_train) / self.batch_size))
max_f1 = 0.0
max_intent_acc = 0.0
for epoch in range(self.num_epochs):
# shuffle train in each epoch
shuffle_index = np.arange(len(X_train))
np.random.shuffle(shuffle_index)
X_train = X_train[shuffle_index]
seq_len_train = seq_len_train[shuffle_index]
y_train = y_train[shuffle_index]
X_train_tag = X_train_tag[shuffle_index]
y_intent_train = y_intent_train[shuffle_index]
print("current epoch: %d" % (epoch))
for iteration in range(num_iterations):
# train
X_train_batch, y_train_batch, X_train_tag_batch, y_intent_train_batch, seq_len_batch_train = \
helper.nextBatch(X_train, y_train, X_train_tag, y_intent_train, seq_len_train, start_index=iteration * self.batch_size, batch_size=self.batch_size)
y_train_weight_batch = 1 + np.array(
(y_train_batch == label2id['B']) |
(y_train_batch == label2id['E']) |
(y_train_batch == label2id['X']) |
(y_train_batch == label2id['Z']) |
(y_train_batch == label2id['U']) |
(y_train_batch == label2id['W']), float)
transition_batch = helper.get_transition(y_train_batch)
if self.crf_flag == 2:
_, loss_train, max_scores, max_scores_pre, predicts_train_intent, len_train = \
sess.run([
self.optimizer,
self.sum_loss,
self.max_scores,
self.max_scores_pre,
self.intent_prediction,
self.sequence_len,
],
feed_dict={
self.targets_transition:transition_batch,
self.inputs:X_train_batch,
self.slot_targets:y_train_batch,
self.targets_weight:y_train_weight_batch,
self.input_tag:X_train_tag_batch,
self.intent_target:y_intent_train_batch,
# self.sequence_len:seq_len_batch_train
})
if iteration % 100 == 0:
predicts_train = self.viterbi(
max_scores,
max_scores_pre,
len_train,
predict_size=self.batch_size)
precision_train, recall_train, f1_train, acc_train = self.evaluate(
X_train_batch, y_train_batch, y_intent_train_batch,
predicts_train, predicts_train_intent, id2char,
id2label)
print(
"iteration, train loss, train precision, train recall, train f1, train acc",
iteration, loss_train, precision_train,
recall_train, f1_train, acc_train)
elif self.crf_flag == 3:
_, transition_params_train, slot_train_logits, loss_train, predicts_train_intent, train_seq_length = \
sess.run([
self.optimizer,
self.transition_params,
self.slot_logits,
self.sum_loss,
self.intent_prediction,
self.sequence_len,
],
feed_dict={
# self.targets_transition: transition_batch,
self.inputs: X_train_batch,
self.slot_targets: y_train_batch,
# self.targets_weight: y_train_weight_batch,
self.input_tag: X_train_tag_batch,
self.intent_target: y_intent_train_batch,
# self.sequence_len: seq_len_batch_train
})
if iteration % 100 == 0:
label_list = []
for logit, seq_len in zip(slot_train_logits,
train_seq_length):
if seq_len == 0: # padding 0 at last of the data
break
viterbi_seq, _ = viterbi_decode(
logit[:seq_len], transition_params_train)
label_list.append(viterbi_seq)
predicts_train = label_list
precision_train, recall_train, f1_train, acc_train = self.evaluate(
X_train_batch, y_train_batch, y_intent_train_batch,
predicts_train, predicts_train_intent, id2char,
id2label)
print(
"iteration, train loss, train precision, train recall, train f1, train acc",
iteration, loss_train, precision_train,
recall_train, f1_train, acc_train)
# validation
if iteration % 200 == 0:
f1_valid_sum = 0.0
acc_valid_sum = 0.0
loss_valid_sum = 0.0
precision_valid_sum = 0.0
recall_valid_sum = 0.0
num_iterations_valid = int(
math.ceil(1.0 * len(X_valid) / model_dev.batch_size))
for ttt in range(num_iterations_valid):
X_valid_batch, y_valid_batch, X_valid_input_tag_batch, y_intent_valid_batch, seq_len_valid_batch = \
helper.nextBatch(X_valid, y_valid, X_valid_tag, y_intent_valid, seq_len_valid, start_index=ttt * model_dev.batch_size, batch_size=model_dev.batch_size)
y_val_weight_batch = 1 + np.array(
(y_valid_batch == label2id['B']) |
(y_valid_batch == label2id['E']) |
(y_valid_batch == label2id['X']) |
(y_valid_batch == label2id['Z']) |
(y_valid_batch == label2id['U']) |
(y_valid_batch == label2id['W']), float)
transition_batch = helper.get_transition(y_valid_batch)
if self.crf_flag == 2:
loss_valid, max_scores, max_scores_pre, predicts_valid_intent, length_dev = \
sess.run([
model_dev.sum_loss,
model_dev.max_scores,
model_dev.max_scores_pre,
model_dev.intent_prediction,
model_dev.sequence_len,
],
feed_dict={
model_dev.targets_transition:transition_batch,
model_dev.inputs:X_valid_batch,
model_dev.slot_targets:y_valid_batch,
model_dev.targets_weight:y_val_weight_batch,
model_dev.input_tag:X_valid_input_tag_batch,
model_dev.intent_target:y_intent_valid_batch,
# model_dev.sequence_len:seq_len_valid_batch
})
predicts_valid = model_dev.viterbi(
max_scores,
max_scores_pre,
length_dev,
predict_size=model_dev.batch_size)
elif self.crf_flag == 3:
slot_train_logits, transition_params_train, length_dev, intent_prediction, loss_valid = \
sess.run([model_dev.slot_logits,
model_dev.transition_params,
model_dev.sequence_len,
model_dev.intent_prediction,
model_dev.sum_loss],
feed_dict={
# model_dev.targets_transition:transition_batch,
model_dev.inputs:X_valid_batch,
model_dev.slot_targets:y_valid_batch,
# model_dev.targets_weight:y_val_weight_batch,
model_dev.input_tag:X_valid_input_tag_batch,
model_dev.intent_target:y_intent_valid_batch,
# model_dev.sequence_len:seq_len_valid_batch
})
label_list = []
for logit, seq_len in zip(slot_train_logits,
length_dev):
if seq_len == 0: # padding 0 at last of the data
break
viterbi_seq, _ = viterbi_decode(
logit[:seq_len], transition_params_train)
label_list.append(viterbi_seq)
predicts_valid = label_list
predicts_valid_intent = intent_prediction
precision_valid, recall_valid, f1_valid, acc_valid = \
model_dev.evaluate(X_valid_batch, y_valid_batch, y_intent_valid_batch, predicts_valid, predicts_valid_intent, id2char, id2label)
f1_valid_sum += f1_valid
acc_valid_sum += acc_valid
loss_valid_sum += loss_valid
precision_valid_sum += precision_valid
recall_valid_sum += recall_valid
if f1_valid_sum > max_f1:
max_f1 = f1_valid_sum
saver.save(sess, "predict_output/model")
if acc_valid_sum > max_intent_acc:
max_intent_acc = acc_valid_sum
# saver.save(sess, "predict_output/model")
print(
"iteration, valid loss, valid precision, valid recall, valid f1, valid acc",
iteration, loss_valid_sum / num_iterations_valid,
precision_valid_sum / num_iterations_valid,
recall_valid_sum / num_iterations_valid,
f1_valid_sum / num_iterations_valid,
acc_valid_sum / num_iterations_valid)
print("max slot f1:", max_f1 / num_iterations_valid)
print("max intent acc", max_intent_acc / num_iterations_valid)
parser.add_argument("-c","--char_emb", help="the char embedding file", default=None)
parser.add_argument("-g","--gpu", help="the id of gpu, the default is 0", default=0, type=int)
args = parser.parse_args()
model_path = args.model_path
test_path = args.test_path
output_path = args.output_path
gpu_config = "/gpu:"+str(args.gpu)
emb_path = args.char_emb
num_steps = 200 # it must consist with the train
start_time = time.time()
helper = Helper()
print ("preparing test data")
helper.char2id, helper.id2char = helper.loadMap("char2id")
helper.label2id, helper.id2label = helper.loadMap("label2id")
x_test, y_test = helper.initFile(inputPath=test_path, seqMaxLen=num_steps)
num_chars = len(helper.id2char.keys())
num_classes = len(helper.id2label.keys())
if emb_path != None:
#embedding_matrix = helper.getEmbedding(emb_path)
embedding_matrix = None
else:
embedding_matrix = None
print("building model")
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
with tf.device(gpu_config):
initializer = tf.random_uniform_initializer(-0.1, 0.1)
def test(self, sess, X_test, X_test_str, X_test_tag, y_intent_test, y_test,
output_path):
char2id, id2char = helper.loadMap("meta_data/char2id")
label2id, id2label = helper.loadMap("meta_data/label2id")
intentlabel2id, intentid2label = helper.loadMap(
"meta_data/intentlabel2id")
num_iterations = int(math.ceil(1.0 * len(X_test) / self.batch_size))
print("number of iteration: " + str(num_iterations))
correct = 0
count = 0
print(len(y_test))
print(len(X_test))
out_intent_file = open("test_output/test_y_intent_out",
"w",
encoding="utf-8")
with open(output_path, mode="w", encoding="utf-8") as outfile:
total_f1 = 0
total_p = 0
total_r = 0
total_acc = 0
for i in range(num_iterations - 1, -1, -1):
#correct=0
#count=0
# print("iteration: " + str(i + 1))
#results = []
results_BME = []
results_XYZ = []
X_test_batch = X_test[i * self.batch_size:(i + 1) *
self.batch_size]
X_test_str_batch = X_test_str[i * self.batch_size:(i + 1) *
self.batch_size]
X_test_tag_batch = X_test_tag[i * self.batch_size:(i + 1) *
self.batch_size]
y_intent_test_batch = y_intent_test[i *
self.batch_size:(i + 1) *
self.batch_size]
y_test_batch = y_test[i * self.batch_size:(i + 1) *
self.batch_size]
if i == num_iterations - 1 and len(
X_test_batch) < self.batch_size:
X_test_batch = list(X_test_batch)
X_test_str_batch = list(X_test_str_batch)
X_test_tag_batch = list(X_test_tag_batch)
y_intent_test_batch = list(y_intent_test_batch)
y_test_batch = list(y_test_batch)
last_size = len(X_test_batch)
X_test_batch += [[0 for j in range(self.num_steps)]
for i in range(self.batch_size -
last_size)]
X_test_str_batch += [['x' for j in range(self.num_steps)]
for i in range(self.batch_size -
last_size)]
X_test_tag_batch += [[[0, 0, 0, 0, 0, 0, 0, 0, 0]
for j in range(self.num_steps)]
for i in range(self.batch_size -
last_size)]
y_intent_test_batch += [
0 for i in range(self.batch_size - last_size)
]
y_test_batch += [[0 for j in range(self.num_steps)]
for i in range(self.batch_size -
last_size)]
X_test_batch = np.array(X_test_batch)
X_test_str_batch = np.array(X_test_str_batch)
X_test_tag_batch = np.array(X_test_tag_batch)
y_intent_test_batch = np.array(y_intent_test_batch)
y_test_batch = np.array(y_test_batch)
results_BME, results_XYZ, results_UVW, y_predictions, correct_batch, count_batch, slot_precision_batch,slot_recall_batch,slot_f1_batch = \
self.predict_batch(sess, X_test_batch, X_test_str_batch, X_test_tag_batch, y_intent_test_batch, y_test_batch, id2label, id2char)
correct += correct_batch
count += count_batch
acc = 1.0 * correct / count
results_BME = results_BME[:last_size]
results_XYZ = results_XYZ[:last_size]
results_UVW = results_UVW[:last_size]
total_f1 += slot_f1_batch
total_p += slot_precision_batch
total_r += slot_recall_batch
total_acc += acc
else:
X_test_batch = np.array(X_test_batch)
X_test_tag_batch = np.array(X_test_tag_batch)
y_intent_test_batch = np.array(y_intent_test_batch)
y_test_batch = np.array(y_test_batch)
results_BME, results_XYZ, results_UVW, y_predictions, correct_batch, count_batch,slot_precision_batch,slot_recall_batch,slot_f1_batch =\
self.predict_batch(sess, X_test_batch, X_test_str_batch, X_test_tag_batch, y_intent_test_batch, y_test_batch, id2label, id2char)
correct += correct_batch
count += count_batch
acc = 1.0 * correct / count
total_f1 += slot_f1_batch
total_p += slot_precision_batch
total_r += slot_recall_batch
total_acc += acc
# print("test intent acc: ", total_acc/num_iterations)
# print("test slot precision: ", total_p/num_iterations)
# print("test slot recall: ", total_r/num_iterations)
# print("test slot f1: ", total_f1/num_iterations)
for j in range(len(y_predictions)):
doc = ''.join(X_test_str_batch[j])
if len(doc) >= 1 and doc[0] != 'x':
out_intent_file.write(
doc + "<@>" + intentid2label[y_predictions[j]] +
"\n")
for i in range(len(results_BME)):
doc = ''.join(X_test_str_batch[i])
outfile.write(doc + "<@>" + " ".join(results_BME[i]) +
"<@>" + " ".join(results_XYZ[i]) + "<@>" +
" ".join(results_UVW[i]) + "\n")
#outfile.write(doc + "<@>" +" ".join(results[i]).encode("utf-8") + "\n")
print("test intent acc: ", total_acc / num_iterations)
print("test slot precision: ", total_p / num_iterations)
print("test slot recall: ", total_r / num_iterations)
total_f1 = 2.0 * total_p * total_r / (total_r + total_p)
print("test slot f1: ", total_f1 / num_iterations)
train_path = 'train.2'
char2id_file = 'char2id'
label2id_file = 'label2id'
save_path = './'
emb_dim = '100'
X_train, y_train, X_val, y_val = helper.getTrain(train_path=train_path,
val_path=None,
seq_max_len=n_steps,
char2id_file=char2id_file,
label2id_file=label2id_file)
sh_index = np.arange(len(X_train))
np.random.shuffle(sh_index)
X_train = X_train[sh_index]
y_train = y_train[sh_index]
char2id, id2char = helper.loadMap(char2id_file)
label2id, id2label = helper.loadMap(label2id_file)
num_chars = len(id2char.keys()) # vocabulary大小
num_classes = len(id2label.keys()) # 标注类别数
emb_path = None
if emb_path != None:
embedding_matrix = helper.getEmbedding(emb_path, char2id_file)
# print len([_ for _ in np.sum(embedding_matrix,axis=1) if _ != 0])
np.savetxt(os.path.join(save_path, "embedding_matrix"), embedding_matrix)
num_chars = embedding_matrix.shape[0] # vocabulary大小
else:
embedding_matrix = None
# char embedding
if embedding_matrix is not None:
embedding = tf.Variable(embedding_matrix,
