def inference(args, config):
infer_text, infer_session, infer_idx, train_pos_idx = inference_data_loader(
args, config)
if os.path.exists(args.infer_out_file):
os.remove(args.infer_out_file)
with tf.variable_scope('Model', reuse=None):
model = SimNet(config, args.encoder_type)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
sess_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True,
log_device_placement=False)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
step = 0
batch_num = 0
checkpoint_file = os.path.join(config['model_dir'], 'model-best')
saver = tf.train.Saver()
saver.restore(sess, checkpoint_file)
infer_batches = batch_iter(infer_idx,
config['batch_size'],
1,
shuffle=False)
pos_fc_out = sess.run(model.query_encoder.fc_out,
feed_dict={model.query: train_pos_idx})
for infer_batch in tqdm(infer_batches):
step += len(infer_batch)
infer_fc_out = sess.run(model.query_encoder.fc_out,
feed_dict={model.query: infer_batch})
for i in range(len(infer_batch)):
feed_dict = {
model.query_vec:
np.repeat(np.expand_dims(infer_fc_out[i], axis=0),
len(train_pos_idx),
axis=0),
model.pos_vec:
pos_fc_out
}
infer_pos_score = sess.run(model.pos_score, feed_dict)
if np.mean(infer_pos_score) > config['threshold']:
pred_label = 1
else:
pred_label = 0
if pred_label == 1:
sample_id = batch_num * config['batch_size'] + i
with open(args.infer_out_file, 'a') as f:
f.write('{}.json,{},{}\n'.format(
infer_session[sample_id],
' '.join(infer_text[sample_id]),
np.mean(infer_pos_score)))
batch_num += 1
sess.close()
paddle.set_device(args.device)
# Loads vocab.
if not os.path.exists(args.vocab_path):
raise RuntimeError('The vocab_path can not be found in the path %s' %
args.vocab_path)
vocab = Vocab.load_vocabulary(
args.vocab_path, unk_token='[UNK]', pad_token='[PAD]')
# Loads dataset.
train_ds, dev_ds, test_ds = load_dataset(
"lcqmc", splits=["train", "dev", "test"])
# Constructs the newtork.
model = SimNet(
network=args.network,
vocab_size=len(vocab),
num_classes=len(train_ds.label_list))
model = paddle.Model(model)
# Reads data and generates mini-batches.
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=vocab.token_to_idx.get('[PAD]', 0)), # query_ids
Pad(axis=0, pad_val=vocab.token_to_idx.get('[PAD]', 0)), # title_ids
Stack(dtype="int64"), # query_seq_lens
Stack(dtype="int64"), # title_seq_lens
Stack(dtype="int64") # label
): [data for data in fn(samples)]
tokenizer = ppnlp.data.JiebaTokenizer(vocab)
trans_fn = partial(convert_example, tokenizer=tokenizer, is_test=False)
train_loader = create_dataloader(
train_ds,
results.extend(labels)
return results
if __name__ == "__main__":
paddle.set_device(args.device)
# Loads vocab.
vocab = Vocab.load_vocabulary(args.vocab_path,
unk_token='[UNK]',
pad_token='[PAD]')
tokenizer = JiebaTokenizer(vocab)
label_map = {0: 'dissimilar', 1: 'similar'}
# Constructs the newtork.
model = SimNet(network=args.network,
vocab_size=len(vocab),
num_classes=len(label_map))
# Loads model parameters.
state_dict = paddle.load(args.params_path)
model.set_dict(state_dict)
print("Loaded parameters from %s" % args.params_path)
# Firstly pre-processing prediction data and then do predict.
data = [
['世界上什么东西最小', '世界上什么东西最小？'],
['光眼睛大就好看吗', '眼睛好看吗？'],
['小蝌蚪找妈妈怎么样', '小蝌蚪找妈妈是谁画的'],
]
examples = preprocess_prediction_data(data, tokenizer)
results = predict(model,
def train(args, config):
seq_length = config['seq_length']
query_train, pos_train, neg_train, query_dev, label_dev, train_pos_idx = train_data_loader(
args, config)
train_datasize = len(query_train)
dev_datasize = len(query_dev)
# train_iterator = train_dataset.make_one_shot_iterator()
# dev_iterator = dev_dataset.make_one_shot_iterator()
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.variable_scope('Model', reuse=None, initializer=init):
model = SimNet(config, args.encoder_type)
# train_init_op = train_iterator.make_initializer(train_dataset)
# dev_init_op = dev_iterator.make_initializer(dev_dataset)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
sess_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True,
log_device_placement=False)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
# sess.run(train_init_op)
num_epoch = 0
step = 0
global_step = tf.Variable(0, name='global_step', trainable=False)
lr = float(config['learning_rate'])
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
grad_and_vars = optimizer.compute_gradients(model.pairwise_hinge_loss)
train_op = optimizer.apply_gradients(grad_and_vars,
global_step=global_step)
checkpoint_dir = os.path.abspath(config['model_dir'])
checkpoint_prefix = os.path.join(checkpoint_dir, 'model')
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables())
sess.run(tf.global_variables_initializer())
train_batches = batch_iter(
list(zip(query_train, pos_train, neg_train)), config['batch_size'],
config['n_epochs'])
val_best_acc = 0.0
for batch in train_batches:
query_batch, pos_batch, neg_batch = zip(*batch)
feed_dict = {
model.query: query_batch,
model.pos_input: pos_batch,
model.neg_input: neg_batch
}
fetches = {
'train_op': train_op,
'train_loss': model.pairwise_hinge_loss,
'step': global_step
}
fetch_vals = sess.run(fetches, feed_dict)
if fetch_vals['step'] % 1000 == 1:
print('step {}, loss {:g}.'.format(fetch_vals['step'],
fetch_vals['train_loss']))
if fetch_vals['step'] % config['val_intervals'] == 0:
print("Evaluating.")
pos_fc_out = sess.run(model.query_encoder.fc_out,
feed_dict={model.query: train_pos_idx})
val_batches = batch_iter(list(zip(query_dev, label_dev)),
config['batch_size'],
1,
shuffle=False)
val_corrects = 0
val_step = 0
for val_batch in tqdm(val_batches):
val_step += len(val_batch)
query_dev_batch, label_dev_batch = zip(*val_batch)
query_fc_out = sess.run(
model.query_encoder.fc_out,
feed_dict={model.query: query_dev_batch})
for i in range(len(val_batch)):
feed_dict = {
model.query_vec:
np.repeat(np.expand_dims(query_fc_out[i], axis=0),
len(train_pos_idx),
axis=0),
model.pos_vec:
pos_fc_out
}
val_pos_score = sess.run(model.pos_score, feed_dict)
if np.mean(val_pos_score) > config['threshold']:
pred_label = 1
else:
pred_label = 0
val_corrects = val_corrects + 1 if pred_label == label_dev_batch[
i] else val_corrects
val_acc = val_corrects * 1.0 / val_step
# saver.save(sess, checkpoint_prefix, fetch_vals['step'])
if val_acc >= val_best_acc:
val_best_acc = val_acc
saver.save(sess, os.path.join(checkpoint_dir,
'model-best'))
print("Better val accuray: {:.4g}%, saving at {}.".format(
val_acc * 100, checkpoint_dir))
else:
print('Val accuray: {:.4g}%'.format(val_acc * 100))
sys.stdout.flush()
sess.close()
def test(args, config):
query_test, label_test, train_pos_idx, test_pos_num, test_neg_num = test_data_loader(
args, config)
test_datasize = len(query_test)
with tf.variable_scope('Model', reuse=None):
model = SimNet(config, args.encoder_type)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
sess_config = tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True,
log_device_placement=False)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
step = 0
checkpoint_file = os.path.join(config['model_dir'], 'model-best')
saver = tf.train.Saver()
saver.restore(sess, checkpoint_file)
test_batches = batch_iter(list(zip(query_test, label_test)),
config['batch_size'],
1,
shuffle=False)
test_accuracy = 0.0
test_precision = 0.0
test_recall = 0.0
test_f1 = 0.0
test_corrects = 0
test_true_pos = 0
test_pred_pos = 0
pos_fc_out = sess.run(model.query_encoder.fc_out,
feed_dict={model.query: train_pos_idx})
for batch in tqdm(test_batches):
step += len(batch)
query_test_batch, label_test_batch = zip(*batch)
query_fc_out = sess.run(model.query_encoder.fc_out,
feed_dict={model.query: query_test_batch})
for i in range(len(batch)):
feed_dict = {
model.query_vec:
np.repeat(np.expand_dims(query_fc_out[i], axis=0),
len(train_pos_idx),
axis=0),
model.pos_vec:
pos_fc_out
}
test_pos_score = sess.run(model.pos_score, feed_dict)
if np.mean(test_pos_score) > config['threshold']:
pred_label = 1
else:
pred_label = 0
test_corrects = test_corrects + 1 if pred_label == label_test_batch[
i] else test_corrects
test_true_pos = test_true_pos + 1 if pred_label == 1 and label_test_batch[
i] == 1 else test_true_pos
test_pred_pos = test_pred_pos + 1 if pred_label == 1 else test_pred_pos
test_accuracy = test_corrects * 1.0 / step
test_precision = test_true_pos * 1.0 / test_pred_pos
test_recall = test_true_pos * 1.0 / test_pos_num
test_f1 = 2 * test_precision * test_recall / (test_precision +
test_recall)
print(
"Test accuracy : {:.4g}%\nprecision: {:.4g}%\nrecall: {:.4g}%\nf1: {:.4g}%"
.format(test_accuracy * 100, test_precision * 100,
test_recall * 100, test_f1 * 100))
sess.close()