def infer(conf_dict, args):
"""
run predict
"""
logging.info("start test process ...")
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
with fluid.dygraph.guard(place):
vocab = utils.load_vocab(args.vocab_path)
simnet_process = reader.SimNetProcessor(args, vocab)
get_infer_examples = simnet_process.get_infer_reader
infer_loader = fluid.io.DataLoader.from_generator(
capacity=16,
return_list=True,
iterable=True,
use_double_buffer=True)
infer_loader.set_sample_list_generator(
paddle.batch(get_infer_examples, batch_size=args.batch_size),
place)
conf_dict['dict_size'] = len(vocab)
conf_dict['seq_len'] = args.seq_len
net = utils.import_class("./nets", conf_dict["net"]["module_name"],
conf_dict["net"]["class_name"])(conf_dict)
model, _ = load_dygraph(args.init_checkpoint)
net.set_dict(model)
pred_list = []
if args.task_mode == "pairwise":
for left, pos_right in infer_loader():
left = fluid.layers.reshape(left, shape=[-1, 1])
pos_right = fluid.layers.reshape(pos_right, shape=[-1, 1])
left_feat, pos_score = net(left, pos_right)
pred = pos_score
pred_list += list(
map(lambda item: str((item[0] + 1) / 2), pred.numpy()))
else:
for left, right in infer_loader():
left = fluid.layers.reshape(left, shape=[-1, 1])
pos_right = fluid.layers.reshape(right, shape=[-1, 1])
left_feat, pred = net(left, right)
pred_list += map(lambda item: str(np.argmax(item)),
pred.numpy())
with io.open(args.infer_result_path, "w",
encoding="utf8") as infer_file:
for _data, _pred in zip(simnet_process.get_infer_data(),
pred_list):
infer_file.write(_data + "\t" + _pred + "\n")
logging.info("infer result saved in %s" %
os.path.join(os.getcwd(), args.infer_result_path))
def infer(args):
"""
run predict
"""
vocab = utils.load_vocab(args.vocab_path)
simnet_process = reader.SimNetProcessor(args, vocab)
# Get model path
model_path = args.init_checkpoint
# Get device
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
# Get executor
executor = fluid.Executor(place=place)
# Load model
program, feed_var_names, fetch_targets = fluid.io.load_inference_model(
model_path, executor)
if args.task_mode == "pairwise":
# Get Feeder and Reader
infer_feeder = fluid.DataFeeder(place=place,
feed_list=feed_var_names,
program=program)
infer_reader = simnet_process.get_infer_reader
else:
# Get Feeder and Reader
infer_feeder = fluid.DataFeeder(place=place,
feed_list=feed_var_names,
program=program)
infer_reader = simnet_process.get_infer_reader
# Get batch data iterator
batch_data = paddle.batch(infer_reader, args.batch_size, drop_last=False)
logging.info("start test process ...")
preds_list = []
for iter, data in enumerate(batch_data()):
output = executor.run(program,
feed=infer_feeder.feed(data),
fetch_list=fetch_targets)
if args.task_mode == "pairwise":
preds_list += list(
map(lambda item: str((item[0] + 1) / 2), output[1]))
else:
preds_list += map(lambda item: str(np.argmax(item)), output[1])
with open(args.infer_result_path, "w") as infer_file:
for _data, _pred in zip(simnet_process.get_infer_data(), preds_list):
infer_file.write(_data + "\t" + _pred + "\n")
logging.info("infer result saved in %s" %
os.path.join(os.getcwd(), args.infer_result_path))
def infer(conf_dict, args):
"""
run predict
"""
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = fluid.Executor(place)
vocab = utils.load_vocab(args.vocab_path)
simnet_process = reader.SimNetProcessor(args, vocab)
startup_prog = fluid.Program()
get_infer_examples = simnet_process.get_infer_reader
batch_data = fluid.io.batch(get_infer_examples,
args.batch_size,
drop_last=False)
test_prog = fluid.Program()
conf_dict['dict_size'] = len(vocab)
net = utils.import_class("../shared_modules/models/matching",
conf_dict["net"]["module_name"],
conf_dict["net"]["class_name"])(conf_dict)
if args.task_mode == "pairwise":
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
infer_loader, left, pos_right = create_model(args,
is_inference=True)
left_feat, pos_score = net.predict(left, pos_right)
pred = pos_score
test_prog = test_prog.clone(for_test=True)
else:
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
infer_loader, left, right = create_model(args,
is_inference=True)
left_feat, pred = net.predict(left, right)
test_prog = test_prog.clone(for_test=True)
exe.run(startup_prog)
utils.init_checkpoint(exe, args.init_checkpoint, main_program=test_prog)
test_exe = exe
infer_loader.set_sample_list_generator(batch_data)
logging.info("start test process ...")
preds_list = []
fetch_list = [pred.name]
output = []
infer_loader.start()
while True:
try:
output = test_exe.run(program=test_prog, fetch_list=fetch_list)
if args.task_mode == "pairwise":
preds_list += list(
map(lambda item: str((item[0] + 1) / 2), output[0]))
else:
preds_list += map(lambda item: str(np.argmax(item)), output[0])
except fluid.core.EOFException:
infer_loader.reset()
break
with io.open(args.infer_result_path, "w", encoding="utf8") as infer_file:
for _data, _pred in zip(simnet_process.get_infer_data(), preds_list):
infer_file.write(_data + "\t" + _pred + "\n")
logging.info("infer result saved in %s" %
os.path.join(os.getcwd(), args.infer_result_path))
def test(conf_dict, args):
"""
Evaluation Function
"""
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = fluid.Executor(place)
vocab = utils.load_vocab(args.vocab_path)
simnet_process = reader.SimNetProcessor(args, vocab)
startup_prog = fluid.Program()
get_test_examples = simnet_process.get_reader("test")
batch_data = fluid.io.batch(get_test_examples,
args.batch_size,
drop_last=False)
test_prog = fluid.Program()
conf_dict['dict_size'] = len(vocab)
net = utils.import_class("../shared_modules/models/matching",
conf_dict["net"]["module_name"],
conf_dict["net"]["class_name"])(conf_dict)
metric = fluid.metrics.Auc(name="auc")
with io.open("predictions.txt", "w", encoding="utf8") as predictions_file:
if args.task_mode == "pairwise":
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_loader, left, pos_right = create_model(
args, is_inference=True)
left_feat, pos_score = net.predict(left, pos_right)
pred = pos_score
test_prog = test_prog.clone(for_test=True)
else:
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_loader, left, right = create_model(args,
is_inference=True)
left_feat, pred = net.predict(left, right)
test_prog = test_prog.clone(for_test=True)
exe.run(startup_prog)
utils.init_checkpoint(exe,
args.init_checkpoint,
main_program=test_prog)
test_exe = exe
test_loader.set_sample_list_generator(batch_data)
logging.info("start test process ...")
test_loader.start()
pred_list = []
fetch_list = [pred.name]
output = []
while True:
try:
output = test_exe.run(program=test_prog, fetch_list=fetch_list)
if args.task_mode == "pairwise":
pred_list += list(
map(lambda item: float(item[0]), output[0]))
predictions_file.write(u"\n".join(
map(lambda item: str((item[0] + 1) / 2), output[0])) +
"\n")
else:
pred_list += map(lambda item: item, output[0])
predictions_file.write(u"\n".join(
map(lambda item: str(np.argmax(item)), output[0])) +
"\n")
except fluid.core.EOFException:
test_loader.reset()
break
if args.task_mode == "pairwise":
pred_list = np.array(pred_list).reshape((-1, 1))
pred_list = (pred_list + 1) / 2
pred_list = np.hstack(
(np.ones_like(pred_list) - pred_list, pred_list))
else:
pred_list = np.array(pred_list)
labels = simnet_process.get_test_label()
metric.update(pred_list, labels)
if args.compute_accuracy:
acc = utils.get_accuracy(pred_list, labels, args.task_mode,
args.lamda)
logging.info("AUC of test is %f, Accuracy of test is %f" %
(metric.eval(), acc))
else:
logging.info("AUC of test is %f" % metric.eval())
if args.verbose_result:
utils.get_result_file(args)
logging.info("test result saved in %s" %
os.path.join(os.getcwd(), args.test_result_path))
def train(conf_dict, args):
"""
train processic
"""
# loading vocabulary
vocab = utils.load_vocab(args.vocab_path)
# get vocab size
conf_dict['dict_size'] = len(vocab)
# Load network structure dynamically
net = utils.import_class("../shared_modules/models/matching",
conf_dict["net"]["module_name"],
conf_dict["net"]["class_name"])(conf_dict)
# Load loss function dynamically
loss = utils.import_class("../shared_modules/models/matching/losses",
conf_dict["loss"]["module_name"],
conf_dict["loss"]["class_name"])(conf_dict)
# Load Optimization method
optimizer = utils.import_class(
"../shared_modules/models/matching/optimizers", "paddle_optimizers",
conf_dict["optimizer"]["class_name"])(conf_dict)
# load auc method
metric = fluid.metrics.Auc(name="auc")
# Get device
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
exe = fluid.Executor(place)
startup_prog = fluid.Program()
train_program = fluid.Program()
# used for continuous evaluation
if args.enable_ce:
SEED = 102
startup_prog.random_seed = SEED
train_program.random_seed = SEED
simnet_process = reader.SimNetProcessor(args, vocab)
if args.task_mode == "pairwise":
# Build network
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
train_loader, left, pos_right, neg_right = create_model(args)
left_feat, pos_score = net.predict(left, pos_right)
pred = pos_score
_, neg_score = net.predict(left, neg_right)
avg_cost = loss.compute(pos_score, neg_score)
avg_cost.persistable = True
optimizer.ops(avg_cost)
# Get Reader
get_train_examples = simnet_process.get_reader("train",
epoch=args.epoch)
if args.do_valid:
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_loader, left, pos_right = create_model(
args, is_inference=True)
left_feat, pos_score = net.predict(left, pos_right)
pred = pos_score
test_prog = test_prog.clone(for_test=True)
else:
# Build network
with fluid.program_guard(train_program, startup_prog):
with fluid.unique_name.guard():
train_loader, left, right, label = create_model(
args, is_pointwise=True)
left_feat, pred = net.predict(left, right)
avg_cost = loss.compute(pred, label)
avg_cost.persistable = True
optimizer.ops(avg_cost)
# Get Feeder and Reader
get_train_examples = simnet_process.get_reader("train",
epoch=args.epoch)
if args.do_valid:
test_prog = fluid.Program()
with fluid.program_guard(test_prog, startup_prog):
with fluid.unique_name.guard():
test_loader, left, right = create_model(args,
is_inference=True)
left_feat, pred = net.predict(left, right)
test_prog = test_prog.clone(for_test=True)
if args.init_checkpoint is not "":
utils.init_checkpoint(exe, args.init_checkpoint, startup_prog)
def valid_and_test(test_program, test_loader, get_valid_examples, process,
mode, exe, fetch_list):
"""
return auc and acc
"""
# Get Batch Data
batch_data = fluid.io.batch(get_valid_examples,
args.batch_size,
drop_last=False)
test_loader.set_sample_list_generator(batch_data)
test_loader.start()
pred_list = []
while True:
try:
_pred = exe.run(program=test_program, fetch_list=[pred.name])
pred_list += list(_pred)
except fluid.core.EOFException:
test_loader.reset()
break
pred_list = np.vstack(pred_list)
if mode == "test":
label_list = process.get_test_label()
elif mode == "valid":
label_list = process.get_valid_label()
if args.task_mode == "pairwise":
pred_list = (pred_list + 1) / 2
pred_list = np.hstack(
(np.ones_like(pred_list) - pred_list, pred_list))
metric.reset()
metric.update(pred_list, label_list)
auc = metric.eval()
if args.compute_accuracy:
acc = utils.get_accuracy(pred_list, label_list, args.task_mode,
args.lamda)
return auc, acc
else:
return auc
# run train
logging.info("start train process ...")
# set global step
global_step = 0
ce_info = []
train_exe = exe
#for epoch_id in range(args.epoch):
# used for continuous evaluation
if args.enable_ce:
train_batch_data = fluid.io.batch(get_train_examples,
args.batch_size,
drop_last=False)
else:
train_batch_data = fluid.io.batch(fluid.io.shuffle(get_train_examples,
buf_size=10000),
args.batch_size,
drop_last=False)
train_loader.set_sample_list_generator(train_batch_data)
train_loader.start()
exe.run(startup_prog)
losses = []
start_time = time.time()
while True:
try:
global_step += 1
fetch_list = [avg_cost.name]
avg_loss = train_exe.run(program=train_program,
fetch_list=fetch_list)
losses.append(np.mean(avg_loss[0]))
if args.do_valid and global_step % args.validation_steps == 0:
get_valid_examples = simnet_process.get_reader("valid")
valid_result = valid_and_test(test_prog, test_loader,
get_valid_examples,
simnet_process, "valid", exe,
[pred.name])
if args.compute_accuracy:
valid_auc, valid_acc = valid_result
logging.info(
"global_steps: %d, valid_auc: %f, valid_acc: %f, valid_loss: %f"
% (global_step, valid_auc, valid_acc, np.mean(losses)))
else:
valid_auc = valid_result
logging.info(
"global_steps: %d, valid_auc: %f, valid_loss: %f" %
(global_step, valid_auc, np.mean(losses)))
if global_step % args.save_steps == 0:
model_save_dir = os.path.join(args.output_dir,
conf_dict["model_path"])
model_path = os.path.join(model_save_dir, str(global_step))
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
if args.task_mode == "pairwise":
feed_var_names = [left.name, pos_right.name]
target_vars = [left_feat, pos_score]
else:
feed_var_names = [
left.name,
right.name,
]
target_vars = [left_feat, pred]
fluid.io.save_inference_model(model_path, feed_var_names,
target_vars, exe, test_prog)
logging.info("saving infer model in %s" % model_path)
except fluid.core.EOFException:
train_loader.reset()
break
end_time = time.time()
#logging.info("epoch: %d, loss: %f, used time: %d sec" %
#(epoch_id, np.mean(losses), end_time - start_time))
ce_info.append([np.mean(losses), end_time - start_time])
#final save
logging.info("the final step is %s" % global_step)
model_save_dir = os.path.join(args.output_dir, conf_dict["model_path"])
model_path = os.path.join(model_save_dir, str(global_step))
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
if args.task_mode == "pairwise":
feed_var_names = [left.name, pos_right.name]
target_vars = [left_feat, pos_score]
else:
feed_var_names = [
left.name,
right.name,
]
target_vars = [left_feat, pred]
fluid.io.save_inference_model(model_path, feed_var_names, target_vars, exe,
test_prog)
logging.info("saving infer model in %s" % model_path)
# used for continuous evaluation
if args.enable_ce:
card_num = get_cards()
ce_loss = 0
ce_time = 0
try:
ce_loss = ce_info[-1][0]
ce_time = ce_info[-1][1]
except:
logging.info("ce info err!")
print("kpis\teach_step_duration_%s_card%s\t%s" %
(args.task_name, card_num, ce_time))
print("kpis\ttrain_loss_%s_card%s\t%f" %
(args.task_name, card_num, ce_loss))
if args.do_test:
if args.task_mode == "pairwise":
# Get Feeder and Reader
get_test_examples = simnet_process.get_reader("test")
else:
# Get Feeder and Reader
get_test_examples = simnet_process.get_reader("test")
test_result = valid_and_test(test_prog, test_loader, get_test_examples,
simnet_process, "test", exe, [pred.name])
if args.compute_accuracy:
test_auc, test_acc = test_result
logging.info("AUC of test is %f, Accuracy of test is %f" %
(test_auc, test_acc))
else:
test_auc = test_result
logging.info("AUC of test is %f" % test_auc)
def train(conf_dict, args):
"""
train process
"""
# Get device
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
# run train
logging.info("start train process ...")
def valid_and_test(pred_list, process, mode):
"""
return auc and acc
"""
pred_list = np.vstack(pred_list)
if mode == "test":
label_list = process.get_test_label()
elif mode == "valid":
label_list = process.get_valid_label()
if args.task_mode == "pairwise":
pred_list = (pred_list + 1) / 2
pred_list = np.hstack(
(np.ones_like(pred_list) - pred_list, pred_list))
metric.reset()
metric.update(pred_list, label_list)
auc = metric.eval()
if args.compute_accuracy:
acc = utils.get_accuracy(pred_list, label_list, args.task_mode,
args.lamda)
return auc, acc
else:
return auc
with fluid.dygraph.guard(place):
# used for continuous evaluation
if args.enable_ce:
SEED = 102
fluid.default_startup_program().random_seed = SEED
fluid.default_main_program().random_seed = SEED
# loading vocabulary
vocab = utils.load_vocab(args.vocab_path)
# get vocab size
conf_dict['dict_size'] = len(vocab)
conf_dict['seq_len'] = args.seq_len
# Load network structure dynamically
net = utils.import_class("./nets", conf_dict["net"]["module_name"],
conf_dict["net"]["class_name"])(conf_dict)
if args.init_checkpoint is not "":
model, _ = load_dygraph(args.init_checkpoint)
net.set_dict(model)
# Load loss function dynamically
loss = utils.import_class("./nets/losses",
conf_dict["loss"]["module_name"],
conf_dict["loss"]["class_name"])(conf_dict)
# Load Optimization method
learning_rate = conf_dict["optimizer"]["learning_rate"]
optimizer_name = conf_dict["optimizer"]["class_name"]
if optimizer_name == 'SGDOptimizer':
optimizer = fluid.optimizer.SGDOptimizer(
learning_rate, parameter_list=net.parameters())
elif optimizer_name == 'AdamOptimizer':
beta1 = conf_dict["optimizer"]["beta1"]
beta2 = conf_dict["optimizer"]["beta2"]
epsilon = conf_dict["optimizer"]["epsilon"]
optimizer = fluid.optimizer.AdamOptimizer(
learning_rate,
beta1=beta1,
beta2=beta2,
epsilon=epsilon,
parameter_list=net.parameters())
# load auc method
metric = fluid.metrics.Auc(name="auc")
simnet_process = reader.SimNetProcessor(args, vocab)
# set global step
global_step = 0
ce_info = []
losses = []
start_time = time.time()
train_pyreader = fluid.io.PyReader(capacity=16,
return_list=True,
use_double_buffer=True)
get_train_examples = simnet_process.get_reader("train",
epoch=args.epoch)
train_pyreader.decorate_sample_list_generator(
paddle.batch(get_train_examples, batch_size=args.batch_size),
place)
if args.do_valid:
valid_pyreader = fluid.io.PyReader(capacity=16,
return_list=True,
use_double_buffer=True)
get_valid_examples = simnet_process.get_reader("valid")
valid_pyreader.decorate_sample_list_generator(
paddle.batch(get_valid_examples, batch_size=args.batch_size),
place)
pred_list = []
if args.task_mode == "pairwise":
for left, pos_right, neg_right in train_pyreader():
left = fluid.layers.reshape(left, shape=[-1, 1])
pos_right = fluid.layers.reshape(pos_right, shape=[-1, 1])
neg_right = fluid.layers.reshape(neg_right, shape=[-1, 1])
net.train()
global_step += 1
left_feat, pos_score = net(left, pos_right)
pred = pos_score
_, neg_score = net(left, neg_right)
avg_cost = loss.compute(pos_score, neg_score)
losses.append(np.mean(avg_cost.numpy()))
avg_cost.backward()
optimizer.minimize(avg_cost)
net.clear_gradients()
if args.do_valid and global_step % args.validation_steps == 0:
for left, pos_right in valid_pyreader():
left = fluid.layers.reshape(left, shape=[-1, 1])
pos_right = fluid.layers.reshape(pos_right,
shape=[-1, 1])
net.eval()
left_feat, pos_score = net(left, pos_right)
pred = pos_score
pred_list += list(pred.numpy())
valid_result = valid_and_test(pred_list, simnet_process,
"valid")
if args.compute_accuracy:
valid_auc, valid_acc = valid_result
logging.info(
"global_steps: %d, valid_auc: %f, valid_acc: %f, valid_loss: %f"
% (global_step, valid_auc, valid_acc,
np.mean(losses)))
else:
valid_auc = valid_result
logging.info(
"global_steps: %d, valid_auc: %f, valid_loss: %f" %
(global_step, valid_auc, np.mean(losses)))
if global_step % args.save_steps == 0:
model_save_dir = os.path.join(args.output_dir,
conf_dict["model_path"])
model_path = os.path.join(model_save_dir, str(global_step))
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
fluid.dygraph.save_dygraph(net.state_dict(), model_path)
logging.info("saving infer model in %s" % model_path)
else:
for left, right, label in train_pyreader():
left = fluid.layers.reshape(left, shape=[-1, 1])
right = fluid.layers.reshape(right, shape=[-1, 1])
label = fluid.layers.reshape(label, shape=[-1, 1])
net.train()
global_step += 1
left_feat, pred = net(left, right)
avg_cost = loss.compute(pred, label)
losses.append(np.mean(avg_cost.numpy()))
avg_cost.backward()
optimizer.minimize(avg_cost)
net.clear_gradients()
if args.do_valid and global_step % args.validation_steps == 0:
for left, right in valid_pyreader():
left = fluid.layers.reshape(left, shape=[-1, 1])
right = fluid.layers.reshape(right, shape=[-1, 1])
net.eval()
left_feat, pred = net(left, right)
pred_list += list(pred.numpy())
valid_result = valid_and_test(pred_list, simnet_process,
"valid")
if args.compute_accuracy:
valid_auc, valid_acc = valid_result
logging.info(
"global_steps: %d, valid_auc: %f, valid_acc: %f, valid_loss: %f"
% (global_step, valid_auc, valid_acc,
np.mean(losses)))
else:
valid_auc = valid_result
logging.info(
"global_steps: %d, valid_auc: %f, valid_loss: %f" %
(global_step, valid_auc, np.mean(losses)))
if global_step % args.save_steps == 0:
model_save_dir = os.path.join(args.output_dir,
conf_dict["model_path"])
model_path = os.path.join(model_save_dir, str(global_step))
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
fluid.dygraph.save_dygraph(net.state_dict(), model_path)
logging.info("saving infer model in %s" % model_path)
end_time = time.time()
ce_info.append([np.mean(losses), end_time - start_time])
# final save
logging.info("the final step is %s" % global_step)
model_save_dir = os.path.join(args.output_dir, conf_dict["model_path"])
model_path = os.path.join(model_save_dir, str(global_step))
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
fluid.dygraph.save_dygraph(net.state_dict(), model_path)
logging.info("saving infer model in %s" % model_path)
# used for continuous evaluation
if args.enable_ce:
# if True:
card_num = get_cards()
ce_loss = 0
ce_time = 0
try:
ce_loss = ce_info[-1][0]
ce_time = ce_info[-1][1]
except:
logging.info("ce info err!")
print("kpis\teach_step_duration_%s_card%s\t%s" %
(args.task_name, card_num, ce_time))
print("kpis\ttrain_loss_%s_card%s\t%f" %
(args.task_name, card_num, ce_loss))
if args.do_test:
# Get Feeder and Reader
test_pyreader = fluid.io.PyReader(capacity=16,
return_list=True,
use_double_buffer=True)
get_test_examples = simnet_process.get_reader("test")
test_pyreader.decorate_sample_list_generator(
paddle.batch(get_test_examples, batch_size=args.batch_size),
place)
pred_list = []
for left, pos_right in test_pyreader():
left = fluid.layers.reshape(left, shape=[-1, 1])
pos_right = fluid.layers.reshape(pos_right, shape=[-1, 1])
net.eval()
left = fluid.layers.reshape(left, shape=[-1, 1])
pos_right = fluid.layers.reshape(pos_right, shape=[-1, 1])
left_feat, pos_score = net(left, pos_right)
pred = pos_score
pred_list += list(pred.numpy())
test_result = valid_and_test(pred_list, simnet_process, "test")
if args.compute_accuracy:
test_auc, test_acc = test_result
logging.info("AUC of test is %f, Accuracy of test is %f" %
(test_auc, test_acc))
else:
test_auc = test_result
logging.info("AUC of test is %f" % test_auc)
def test(conf_dict, args):
"""
Evaluation Function
"""
logging.info("start test process ...")
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
with fluid.dygraph.guard(place):
vocab = utils.load_vocab(args.vocab_path)
simnet_process = reader.SimNetProcessor(args, vocab)
test_pyreader = fluid.io.PyReader(capacity=16,
return_list=True,
use_double_buffer=True)
get_test_examples = simnet_process.get_reader("test")
test_pyreader.decorate_sample_list_generator(
paddle.batch(get_test_examples, batch_size=args.batch_size), place)
conf_dict['dict_size'] = len(vocab)
conf_dict['seq_len'] = args.seq_len
net = utils.import_class("./nets", conf_dict["net"]["module_name"],
conf_dict["net"]["class_name"])(conf_dict)
model, _ = load_dygraph(args.init_checkpoint)
net.set_dict(model)
metric = fluid.metrics.Auc(name="auc")
pred_list = []
with io.open("predictions.txt", "w",
encoding="utf8") as predictions_file:
if args.task_mode == "pairwise":
for left, pos_right in test_pyreader():
left = fluid.layers.reshape(left, shape=[-1, 1])
pos_right = fluid.layers.reshape(pos_right, shape=[-1, 1])
left_feat, pos_score = net(left, pos_right)
pred = pos_score
# pred_list += list(pred.numpy())
pred_list += list(
map(lambda item: float(item[0]), pred.numpy()))
predictions_file.write(u"\n".join(
map(lambda item: str(
(item[0] + 1) / 2), pred.numpy())) + "\n")
else:
for left, right in test_pyreader():
left = fluid.layers.reshape(left, shape=[-1, 1])
right = fluid.layers.reshape(right, shape=[-1, 1])
left_feat, pred = net(left, right)
# pred_list += list(pred.numpy())
pred_list += list(
map(lambda item: float(item[0]), pred.numpy()))
predictions_file.write(u"\n".join(
map(lambda item: str(np.argmax(item)), pred.numpy())) +
"\n")
if args.task_mode == "pairwise":
pred_list = np.array(pred_list).reshape((-1, 1))
pred_list = (pred_list + 1) / 2
pred_list = np.hstack(
(np.ones_like(pred_list) - pred_list, pred_list))
else:
pred_list = np.array(pred_list)
labels = simnet_process.get_test_label()
metric.update(pred_list, labels)
if args.compute_accuracy:
acc = utils.get_accuracy(pred_list, labels, args.task_mode,
args.lamda)
logging.info("AUC of test is %f, Accuracy of test is %f" %
(metric.eval(), acc))
else:
logging.info("AUC of test is %f" % metric.eval())
if args.verbose_result:
utils.get_result_file(args)
logging.info("test result saved in %s" %
os.path.join(os.getcwd(), args.test_result_path))
def train(conf_dict, args):
"""
train processic
"""
if args.enable_ce:
SEED = 102
fluid.default_startup_program().random_seed = SEED
fluid.default_main_program().random_seed = SEED
# loading vocabulary
vocab = utils.load_vocab(args.vocab_path)
# get vocab size
conf_dict['dict_size'] = len(vocab)
# Get data layer
data = layers.DataLayer()
# Load network structure dynamically
net = utils.import_class("../models/matching",
conf_dict["net"]["module_name"],
conf_dict["net"]["class_name"])(conf_dict)
# Load loss function dynamically
loss = utils.import_class("../models/matching/losses",
conf_dict["loss"]["module_name"],
conf_dict["loss"]["class_name"])(conf_dict)
# Load Optimization method
optimizer = utils.import_class(
"../models/matching/optimizers", "paddle_optimizers",
conf_dict["optimizer"]["class_name"])(conf_dict)
# load auc method
metric = fluid.metrics.Auc(name="auc")
# Get device
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
simnet_process = reader.SimNetProcessor(args, vocab)
if args.task_mode == "pairwise":
# Build network
left = data.ops(name="left", shape=[1], dtype="int64", lod_level=1)
pos_right = data.ops(name="right",
shape=[1],
dtype="int64",
lod_level=1)
neg_right = data.ops(name="neg_right",
shape=[1],
dtype="int64",
lod_level=1)
left_feat, pos_score = net.predict(left, pos_right)
# Get Feeder and Reader
train_feeder = fluid.DataFeeder(
place=place, feed_list=[left.name, pos_right.name, neg_right.name])
train_reader = simnet_process.get_reader("train")
if args.do_valid:
valid_feeder = fluid.DataFeeder(
place=place, feed_list=[left.name, pos_right.name])
valid_reader = simnet_process.get_reader("valid")
pred = pos_score
# Save Infer model
infer_program = fluid.default_main_program().clone(for_test=True)
_, neg_score = net.predict(left, neg_right)
avg_cost = loss.compute(pos_score, neg_score)
avg_cost.persistable = True
else:
# Build network
left = data.ops(name="left", shape=[1], dtype="int64", lod_level=1)
right = data.ops(name="right", shape=[1], dtype="int64", lod_level=1)
label = data.ops(name="label", shape=[1], dtype="int64", lod_level=0)
left_feat, pred = net.predict(left, right)
# Get Feeder and Reader
train_feeder = fluid.DataFeeder(
place=place, feed_list=[left.name, right.name, label.name])
train_reader = simnet_process.get_reader("train")
if args.do_valid:
valid_feeder = fluid.DataFeeder(place=place,
feed_list=[left.name, right.name])
valid_reader = simnet_process.get_reader("valid")
# Save Infer model
infer_program = fluid.default_main_program().clone(for_test=True)
avg_cost = loss.compute(pred, label)
avg_cost.persistable = True
# operate Optimization
optimizer.ops(avg_cost)
executor = fluid.Executor(place)
executor.run(fluid.default_startup_program())
# Get and run executor
parallel_executor = fluid.ParallelExecutor(
use_cuda=args.use_cuda,
loss_name=avg_cost.name,
main_program=fluid.default_main_program())
# Get device number
device_count = parallel_executor.device_count
logging.info("device count: %d" % device_count)
def valid_and_test(program, feeder, reader, process, mode="test"):
"""
return auc and acc
"""
# Get Batch Data
batch_data = paddle.batch(reader, args.batch_size, drop_last=False)
pred_list = []
for data in batch_data():
_pred = executor.run(program=program,
feed=feeder.feed(data),
fetch_list=[pred.name])
pred_list += list(_pred)
pred_list = np.vstack(pred_list)
if mode == "test":
label_list = process.get_test_label()
elif mode == "valid":
label_list = process.get_valid_label()
if args.task_mode == "pairwise":
pred_list = (pred_list + 1) / 2
pred_list = np.hstack(
(np.ones_like(pred_list) - pred_list, pred_list))
metric.reset()
metric.update(pred_list, label_list)
auc = metric.eval()
if args.compute_accuracy:
acc = utils.get_accuracy(pred_list, label_list, args.task_mode,
args.lamda)
return auc, acc
else:
return auc
# run train
logging.info("start train process ...")
# set global step
global_step = 0
ce_info = []
for epoch_id in range(args.epoch):
losses = []
# Get batch data iterator
train_batch_data = paddle.batch(paddle.reader.shuffle(train_reader,
buf_size=10000),
args.batch_size,
drop_last=False)
start_time = time.time()
for iter, data in enumerate(train_batch_data()):
if len(data) < device_count:
logging.info(
"the size of batch data is less than device_count(%d)" %
device_count)
continue
global_step += 1
avg_loss = parallel_executor.run([avg_cost.name],
feed=train_feeder.feed(data))
if args.do_valid and global_step % args.validation_steps == 0:
valid_result = valid_and_test(program=infer_program,
feeder=valid_feeder,
reader=valid_reader,
process=simnet_process,
mode="valid")
if args.compute_accuracy:
valid_auc, valid_acc = valid_result
logging.info(
"global_steps: %d, valid_auc: %f, valid_acc: %f" %
(global_step, valid_auc, valid_acc))
else:
valid_auc = valid_result
logging.info("global_steps: %d, valid_auc: %f" %
(global_step, valid_auc))
if global_step % args.save_steps == 0:
model_save_dir = os.path.join(args.output_dir,
conf_dict["model_path"])
model_path = os.path.join(model_save_dir, str(global_step))
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
if args.task_mode == "pairwise":
feed_var_names = [left.name, pos_right.name]
target_vars = [left_feat, pos_score]
else:
feed_var_names = [
left.name,
right.name,
]
target_vars = [left_feat, pred]
fluid.io.save_inference_model(model_path, feed_var_names,
target_vars, executor,
infer_program)
logging.info("saving infer model in %s" % model_path)
losses.append(np.mean(avg_loss[0]))
end_time = time.time()
logging.info("epoch: %d, loss: %f, used time: %d sec" %
(epoch_id, np.mean(losses), end_time - start_time))
ce_info.append([np.mean(losses), end_time - start_time])
if args.enable_ce:
card_num = get_cards()
ce_loss = 0
ce_time = 0
try:
ce_loss = ce_info[-2][0]
ce_time = ce_info[-2][1]
except:
logging.info("ce info err!")
print("kpis\teach_step_duration_%s_card%s\t%s" %
(args.task_name, card_num, ce_time))
print("kpis\ttrain_loss_%s_card%s\t%f" %
(args.task_name, card_num, ce_loss))
if args.do_test:
if args.task_mode == "pairwise":
# Get Feeder and Reader
test_feeder = fluid.DataFeeder(
place=place, feed_list=[left.name, pos_right.name])
test_reader = simnet_process.get_reader("test")
else:
# Get Feeder and Reader
test_feeder = fluid.DataFeeder(place=place,
feed_list=[left.name, right.name])
test_reader = simnet_process.get_reader("test")
test_result = valid_and_test(program=infer_program,
feeder=test_feeder,
reader=test_reader,
process=simnet_process,
mode="test")
if args.compute_accuracy:
test_auc, test_acc = test_result
logging.info("AUC of test is %f, Accuracy of test is %f" %
(test_auc, test_acc))
else:
test_auc = test_result
logging.info("AUC of test is %f" % test_auc)
def test(conf_dict, args):
"""
run predict
"""
vocab = utils.load_vocab(args.vocab_path)
simnet_process = reader.SimNetProcessor(args, vocab)
# load auc method
metric = fluid.metrics.Auc(name="auc")
with open("predictions.txt", "w") as predictions_file:
# Get model path
model_path = args.init_checkpoint
# Get device
if args.use_cuda:
place = fluid.CUDAPlace(0)
else:
place = fluid.CPUPlace()
# Get executor
executor = fluid.Executor(place=place)
# Load model
program, feed_var_names, fetch_targets = fluid.io.load_inference_model(
model_path, executor)
if args.task_mode == "pairwise":
# Get Feeder and Reader
feeder = fluid.DataFeeder(place=place,
feed_list=feed_var_names,
program=program)
test_reader = simnet_process.get_reader("test")
else:
# Get Feeder and Reader
feeder = fluid.DataFeeder(place=place,
feed_list=feed_var_names,
program=program)
test_reader = simnet_process.get_reader("test")
# Get batch data iterator
batch_data = paddle.batch(test_reader,
args.batch_size,
drop_last=False)
logging.info("start test process ...")
pred_list = []
for iter, data in enumerate(batch_data()):
output = executor.run(program,
feed=feeder.feed(data),
fetch_list=fetch_targets)
if args.task_mode == "pairwise":
pred_list += list(map(lambda item: float(item[0]), output[1]))
predictions_file.write("\n".join(
map(lambda item: str((item[0] + 1) / 2), output[1])) +
"\n")
else:
pred_list += map(lambda item: item, output[1])
predictions_file.write("\n".join(
map(lambda item: str(np.argmax(item)), output[1])) + "\n")
if args.task_mode == "pairwise":
pred_list = np.array(pred_list).reshape((-1, 1))
pred_list = (pred_list + 1) / 2
pred_list = np.hstack(
(np.ones_like(pred_list) - pred_list, pred_list))
else:
pred_list = np.array(pred_list)
labels = simnet_process.get_test_label()
metric.update(pred_list, labels)
if args.compute_accuracy:
acc = utils.get_accuracy(pred_list, labels, args.task_mode,
args.lamda)
logging.info("AUC of test is %f, Accuracy of test is %f" %
(metric.eval(), acc))
else:
logging.info("AUC of test is %f" % metric.eval())
if args.verbose_result:
utils.get_result_file(args)
logging.info("test result saved in %s" %
os.path.join(os.getcwd(), args.test_result_path))
