def train(self, train_data, dev_data):
hparams = self.hparams # 超参数
sess = self.sess # sess
assert len(train_data[0]) == len(
train_data[1]), "Size of features data must be equal to label"
"""
训练
"""
for epoch in range(hparams.epoch):
info = {}
info['loss'] = []
info['norm'] = []
start_time = time.time()
for idx in range(math.ceil(
len(train_data[0]) / hparams.batch_size)):
"""
train_data[0] 是数据， train_data[1]是label
"""
# 截取一个batch的数据，最后一个batch的大小可能不满batch_size
batch = train_data[0][idx * hparams.batch_size:min(
(idx + 1) * hparams.batch_size, len(train_data[0]))]
# 对每一个特征值做hash(类似于做one-hot)
batch = utils.hash_batch(batch, hparams)
# 截取一个batch的label，最后一个batch的大小可能不满batch_size
label = train_data[1][idx * hparams.batch_size:min(
(idx + 1) * hparams.batch_size, len(train_data[1]))]
loss,_,norm=sess.run([self.loss,self.update,self.grad_norm],feed_dict=\
{self.features:batch,self.label:label,self.use_norm:True,
self.second_order_dropout:hparams.second_order_dropout,
self.dnn_dropout:hparams.dnn_dropout
})
info['loss'].append(loss) # 损失
info['norm'].append(norm)
if (idx + 1) % hparams.num_display_steps == 0:
info['learning_rate'] = hparams.learning_rate
info["train_ppl"] = np.mean(info['loss'])
info["avg_grad_norm"] = np.mean(info['norm'])
utils.print_step_info(" ", epoch, idx + 1, info)
del info
info = {}
info['loss'] = []
info['norm'] = []
"""eval时如果模型效果好会保存模型"""
if (idx + 1) % hparams.num_eval_steps == 0 and dev_data:
T = (time.time() - start_time)
self.eval(T, dev_data, hparams, sess)
self.saver.restore(sess, 'model_tmp/model')
T = (time.time() - start_time)
self.eval(T, dev_data, hparams, sess)
os.system("rm -r model_tmp")
def train(self, train_data, dev_data=None):
"""
train_data: (num_train_data * field*size, num_train_data *1)
dev_data: (num_dev_data * field*size, num_dev_data * field*size)
"""
hparams = self.hparams
sess = self.sess
assert len(train_data[0]) == len(train_data[1])
for epoch in range(hparams.epoch):
info = {}
info['loss'] = []
info['norm'] = []
start_time = time.time()
for idx in range(math.ceil(
len(train_data[0]) / hparams.batch_size)):
"""分batch,并针对每个特征值做hash"""
batch=train_data[0][idx*hparams.batch_size:\
min((idx+1)*hparams.batch_size,len(train_data[0]))]
batch = utils.hash_batch(batch, hparams) # hash
label=train_data[1][idx*hparams.batch_size:\
min((idx+1)*hparams.batch_size,len(train_data[1]))]
"""训练"""
loss,_,norm=sess.run([self.loss,self.update,self.grad_norm],\
feed_dict={self.features:batch,self.label:label})
"""打印一些信息"""
info['loss'].append(loss)
info['norm'].append(norm)
if (idx + 1) % hparams.num_display_steps == 0:
info['learning_rate'] = hparams.learning_rate
info["train_ppl"] = np.mean(info['loss'])
info["avg_grad_norm"] = np.mean(info['norm'])
utils.print_step_info(" ", epoch, idx + 1, info)
del info
info = {}
info['loss'] = []
info['norm'] = []
if (idx + 1) % hparams.num_eval_steps == 0 and dev_data:
T = (time.time() - start_time)
self.eval(T, dev_data, hparams, sess)
"""保存模型"""
self.saver.restore(sess, 'model_tmp/model')
T = (time.time() - start_time)
self.eval(T, dev_data, hparams, sess)
os.system("rm -r model_tmp")
def train(hparams):
if hparams.pretrain:
hparams.learning_rate = 0.001
config_proto = tf.ConfigProto(log_device_placement=0,
allow_soft_placement=0)
config_proto.gpu_options.allow_growth = True
train_graph = tf.Graph()
infer_graph = tf.Graph()
with train_graph.as_default():
train_model = Model(hparams, tf.contrib.learn.ModeKeys.TRAIN)
train_sess = tf.Session(graph=train_graph, config=config_proto)
train_sess.run(tf.global_variables_initializer())
train_sess.run(tf.tables_initializer())
with infer_graph.as_default():
infer_model = Model(hparams, tf.contrib.learn.ModeKeys.INFER)
infer_sess = tf.Session(graph=infer_graph, config=config_proto)
infer_sess.run(tf.global_variables_initializer())
infer_sess.run(tf.tables_initializer())
train_model.pretrain_saver.restore(train_sess, 'pretrain_model/best_model')
decay = 0
pay_attention = 0
global_step = 0
train_loss = 0
train_norm = 0
best_score = 1000
epoch = 0
flag = False
if hparams.pretrain:
train_iterator = data_iterator.TextIterator('train', hparams, 32,
'pre_data/train.csv')
dev_iterator = data_iterator.TextIterator('dev', hparams, 512,
'pre_data/dev.csv')
test_iterator = data_iterator.TextIterator('test', hparams, 512,
'pre_data/test.csv')
while True:
start_time = time.time()
try:
cost, _, norm = train_model.train(train_sess, train_iterator)
global_step += 1
train_loss += cost
train_norm += norm
except StopIteration:
continue
if global_step % hparams.num_display_steps == 0:
info = {}
info['learning_rate'] = hparams.learning_rate
info["avg_step_time"] = (
time.time() - start_time) / hparams.num_display_steps
start_time = time.time()
info["train_ppl"] = train_loss / hparams.num_display_steps
info["avg_grad_norm"] = train_norm / hparams.num_display_steps
train_loss = 0
train_norm = 0
utils.print_step_info(" ", global_step, info)
if global_step % hparams.num_eval_steps == 0:
train_model.saver.save(train_sess, 'pretrain_model/model')
with infer_graph.as_default():
infer_model.saver.restore(infer_sess,
'pretrain_model/model')
loss = []
while True:
try:
cost = infer_model.pretrain_infer(
infer_sess, dev_iterator)
loss.append(cost)
except StopIteration:
break
logloss = round(np.mean(loss), 5)
if logloss < best_score:
best_score = logloss
pay_attention = 0
print('logloss', logloss)
print('best logloss', best_score)
print('saving best model')
train_model.pretrain_saver.save(
train_sess, 'pretrain_model/best_model')
else:
pay_attention += 1
print('logloss', logloss)
print('best logloss', best_score)
if pay_attention == hparams.pay_attention:
exit()
train_iterator = data_iterator.TextIterator(
'train', hparams, hparams.batch_size,
'pre_data/expend.csv' if hparams.expend else 'pre_data/train.csv')
dev_iterator = data_iterator.TextIterator('dev', hparams,
hparams.batch_size,
'pre_data/dev.csv')
test_iterator = data_iterator.TextIterator('test', hparams,
hparams.batch_size,
'pre_data/test.csv')
dev_df = pd.read_csv('pre_data/dev.csv')
test_df = pd.read_csv('pre_data/test.csv')
while epoch < hparams.epoch:
start_time = time.time()
try:
cost, _, norm = train_model.train(train_sess, train_iterator)
global_step += 1
train_loss += cost
train_norm += norm
except StopIteration:
epoch += 1
flag = True
if global_step % hparams.num_display_steps == 0:
info = {}
info['learning_rate'] = hparams.learning_rate
info["avg_step_time"] = (time.time() -
start_time) / hparams.num_display_steps
start_time = time.time()
info["train_ppl"] = train_loss / hparams.num_display_steps
info["avg_grad_norm"] = train_norm / hparams.num_display_steps
train_loss = 0
train_norm = 0
utils.print_step_info(" ", global_step, info)
if flag or global_step % hparams.num_eval_steps == 0:
print(epoch)
flag = False
saver = train_model.saver
saver.save(train_sess, 'model_' + hparams.model_name + '/model')
with infer_graph.as_default():
infer_model.saver.restore(
infer_sess, 'model_' + hparams.model_name + '/model')
dev_iterator.reset()
probs = []
while True:
try:
prob = infer_model.infer(infer_sess, dev_iterator)
probs += list(prob)
except StopIteration:
break
dev_df['y_pred'] = probs
try:
logloss = log_loss(dev_df['label'],
dev_df['y_pred'],
eps=1e-15)
except:
break
if logloss < best_score:
best_score = logloss
pay_attention = 0
print('saving best model')
saver.save(train_sess,
'model_' + hparams.model_name + '/best_model')
else:
pay_attention += 1
if pay_attention == hparams.pay_attention:
train_model.saver.restore(
train_sess,
'model_' + hparams.model_name + '/best_model')
pay_attention = 0
decay += 1
hparams.learning_rate /= 2.0
train_model.dey_lrate(train_sess,
hparams.learning_rate)
if decay == hparams.decay_cont:
break
print('logloss', logloss)
print('best logloss', best_score)
with infer_graph.as_default():
infer_model.saver.restore(
infer_sess, 'model_' + hparams.model_name + '/best_model')
dev_iterator.reset()
probs = []
while True:
try:
prob = infer_model.infer(infer_sess, dev_iterator)
probs += list(prob)
except StopIteration:
break
dev_df['y_pre'] = probs
print(log_loss(dev_df['label'], dev_df['y_pre'], eps=1e-15))
dev_df[['y_pre']].to_csv('result/dev_' + hparams.sub_name + '.csv',
index=False)
test_iterator.reset()
probs = []
while True:
try:
prob = infer_model.infer(infer_sess, test_iterator)
probs += list(prob)
except StopIteration:
break
test_df['y_pre'] = probs
test_df[['y_pre']].to_csv('result/test_' + hparams.sub_name + '.csv',
index=False)