def train(self):
# 获得目标损失
token_ids, token_ids_reverse, next_token_ids, next_token_ids_reverse = self.transform.ops(
)
lstm_outputs = self.net.ops(token_ids['token_ids'],
token_ids_reverse['token_ids_reverse'])
next_ids = [
next_token_ids['next_token_ids'],
next_token_ids_reverse['next_token_ids_reverse']
]
train_loss = self.loss.ops(lstm_outputs, next_ids)
dev_config = copy.deepcopy(self.config)
dev_config.update({
"num_epochs": self.dev_epochs,
"train_file": dev_config["dev_file"]
})
dev_transform = clazz(dev_config, 'transform_py',
'transform_class')(dev_config)
dev_ids, dev_ids_reverse, next_dev_ids, next_dev_ids_reverse = dev_transform.ops(
)
dev_lstm_outputs = self.net.ops(dev_ids['token_ids'],
dev_ids_reverse['token_ids_reverse'])
dev_next_ids = [
next_dev_ids['next_token_ids'],
next_dev_ids_reverse['next_token_ids_reverse']
]
dev_loss = self.loss.ops(dev_lstm_outputs, dev_next_ids)
# train
self.trainer.train(self.config, train_loss, -1 * dev_loss)
return True
def load(self, conf_path):
conf_file = conf_path
self.config = load_config(conf_file)
self.dev_epochs = int(
int(self.config["data_size"]) * self.config['num_epochs'] /
int(self.config["batch_size"])) * self.config['dev_data_size']
#训练集的最大批次数（尽可能大）
self.options = {
'bidirectional': self.config['bidirectional'],
'dropout': self.config['dropout'],
'lstm': {
'dim': self.config['dim'],
'n_layers': self.config['n_layers'],
'proj_clip': self.config['proj_clip'],
'cell_clip': self.config['cell_clip'],
'projection_dim': self.config['projection_dim'], # 单个tok 向量的维度
'use_skip_connections': self.config['use_skip_connections']
},
'all_clip_norm_val': self.config['all_clip_norm_val'],
# 轮数
'n_epochs': self.config['num_epochs'],
# number of tokens in training data (this for 1B Word Benchmark)
'n_train_tokens': self.config['n_train_tokens'],
'batch_size': self.config['batch_size'],
'n_tokens_vocab': self.config['n_tokens_vocab'],
'unroll_steps': self.config['unroll_steps'],
'n_negative_samples_batch':
self.config['n_negative_samples_batch'],
'word_emb_file': self.config['word_emb_file']
}
with open(os.path.join(self.config['save_json_dir']), 'w') as fout:
fout.write(json.dumps(self.options))
# 选择类进行组装
# dataset = clazz(self.config, 'dataset_py', 'dataset_class')(self.config)
self.transform = clazz(self.config, 'transform_py',
'transform_class')(self.config)
self.net = clazz(self.config, 'net_py', 'net_class')(self.config)
self.loss = clazz(self.config, "loss_py", "loss_class")(self.config)
self.optimizer = clazz(self.config, 'optimizer_py',
'optimizer_class')(self.config)
self.trainer = clazz(self.config, "trainer_py",
"trainer_class")(self.config, self.transform,
self.net, self.loss,
self.optimizer)
def predict(self, config, label_index, pred):
"""
predict
"""
config.update({
"num_epochs": "1",
"batch_size": "1",
"shuffle": "0",
"train_file": config["test_file"]
})
transform = clazz(config, 'transform_py', 'transform_class')(config)
if self.mode == "pointwise":
test_l, test_r, label = transform.ops()
# test network
pred = self.net.predict(test_l, test_r)
elif self.mode == "pairwise":
test_l, test_r, label = transform.ops()
# test network
pred = self.net.predict(test_l, test_r)
elif self.mode == 'listwise':
input_l, label = transform.ops()
pred = self.net.predict(input_l, input_l)
else:
print(sys.stderr, "training mode not supported")
sys.exit(1)
mean_acc = 0.0
saver = tf.train.Saver()
label_index = tf.argmax(label, 1)
if self.mode == "pointwise":
pred_prob = tf.nn.softmax(pred, -1)
score = tf.reduce_max(pred_prob, -1)
pred_index = tf.argmax(pred_prob, 1)
correct_pred = tf.equal(pred_index, label_index)
acc = tf.reduce_mean(tf.cast(correct_pred, "float"))
elif self.mode == "pairwise":
score = pred
pred_index = tf.argmax(pred, 1)
acc = tf.constant([0.0])
elif self.mode == 'listwise':
pred_prob = tf.nn.softmax(pred, -1)
score = tf.reduce_max(pred_prob, -1)
pred_index = tf.argmax(pred_prob, 1)
correct_pred = tf.equal(pred_index, label_index)
acc = tf.reduce_mean(tf.cast(correct_pred, "float"))
correct_pred = tf.equal(pred_index, label_index)
acc = tf.reduce_mean(tf.cast(correct_pred, "float"))
modelfile = config["test_model_file"]
result_file = open(config["test_result"], "w")
step = 0
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=1)) \
as sess:
sess.run(init)
saver.restore(sess, modelfile)
coord = tf.train.Coordinator()
read_thread = tf.train.start_queue_runners(sess=sess, coord=coord)
while not coord.should_stop():
step += 1
try:
ground, pi, a, prob = sess.run(
[label_index, pred_index, acc, score])
mean_acc += a
for i in range(len(prob)):
result_file.write(
"%s\t%d\t%d\t%f\t%s\n" %
('query', ground[i], pi[i], prob[i], 'info'))
except tf.errors.OutOfRangeError:
coord.request_stop()
coord.join(read_thread)
sess.close()
result_file.close()