def eval(model, optimizer, criterion, post_process, validate_loader, output_path):
"""
:param model:
:param criterion:
:param post_process:
:param validate_loader:
:return:
"""
if not os.path.exists(output_path):
os.makedirs(output_path)
model.eval()
with torch.no_grad():
P, R, F1 =0, 0, 0
for i, batch in enumerate(validate_loader):
pbar = Progbar(target=len(validate_loader))
# 数据进行转换和丢到gpu
batch_size, _, img_h, img_w = batch['img'].shape
for key, value in batch.items():
if value is not None:
if isinstance(value, torch.Tensor):
batch[key] = value.cuda()
preds = model(batch['img'])
batch['shape'] = [(img_h, img_w)]*batch_size
#(batch, nums, 4, 2)
pred_boxes_batch, scores_batch = post_process(batch, preds, is_output_polygon=False)
# print('boxes_batch.shape', boxes_batch)
batch_pred_masks = get_mask(pred_boxes_batch, img_h, img_w)
precision, recall, f1 = get_eval_f1_score(batch_pred_masks, batch['gt'])
P += precision
R += recall
F1 += f1
pbar.update(i + 1, values=[('P', P/(i+1)), ('R', R/(i+1)), ('F1', F1/(i+1))])
save_model(model, optimizer, model_path=os.path.join(output_path,
'model_{}.pth'.format(str(round(F1/(i+1), 2)))),
distributed=False)
def predict(self, sess, dev, labels):
predicted_sentences = []
dev_processed, labels_processed = self.preprocess_sequence_data(
dev, labels)
prog = Progbar(target=1 +
int(len(dev_processed) / self.config.batch_size))
# predict in batches
for i, (enc_batch, dec_batch) in enumerate(
minibatches(dev_processed,
labels_processed,
self.config.batch_size,
shuffle=False)):
#print enc_batch.shape, dec_batch.shape
feed = self.create_feed_dict(enc_batch, dec_batch)
outputs = sess.run(self.test_op, feed_dict=feed)
for j in range(outputs.shape[0]):
sentence = [self.labels_vocab.id2tok[k] for k in outputs[j, :]]
if "</s>" in sentence:
sentence = sentence[:sentence.index("</s>")]
predicted = " ".join(sentence)
print predicted
predicted_sentences.append(predicted)
prog.update(i + 1)
return predicted_sentences
def run_epoch(self, sess, train_examples, dev_set, train_examples_raw, dev_set_raw, epoch, last_epoch):
prog = Progbar(target=1 + int(len(train_examples) / self.config.batch_size))
curr_loss = 0.
num_encountered = 0
for i, batch in enumerate(minibatches(train_examples, self.config.batch_size)):
loss = self.train_on_batch(sess, *batch)
curr_loss += loss
prog.update(i + 1, [("train loss", loss)])
if self.report: self.report.log_train_loss(loss)
num_encountered +=1
# print curr_loss/num_encountered
losses.append(curr_loss/num_encountered)
epochs.append(epoch+1)
print("")
logger.info("Evaluating on development data")
token_cm, metrics = self.evaluate(sess, dev_set, dev_set_raw)
if last_epoch:
self.outputConfusionMatrix(token_cm.as_data())
logger.debug("Token-level confusion matrix:\n" + token_cm.as_table())
logger.debug("Token-level scores:\n" + token_cm.summary())
logger.info("Accuracy: %.2f", metrics[1])
logger.info("Error: %.2f", metrics[0])
return metrics[0], metrics[1]
def run_epoch(self, sess, train_examples, dev_set, train_examples_raw,
dev_set_raw):
global losses
global cm_latest
prog = Progbar(target=1 +
int(len(train_examples) / self.config.batch_size))
curr_loss = 0.
num_encountered = 0
minibatches = generate_minibatches(train_examples,
self.config.batch_size, 1)
for i, batch in enumerate(minibatches):
loss = self.train_on_batch(sess, *batch)
prog.update(i + 1, [("train loss", loss)])
curr_loss += loss
num_encountered += 1
if self.report: self.report.log_train_loss(loss)
losses.append(curr_loss / num_encountered)
print("")
logger.info("Evaluating on development data")
token_cm, metrics = self.evaluate(sess, dev_set, dev_set_raw)
cm_latest = token_cm
logger.debug("Token-level confusion matrix:\n" + token_cm.as_table())
logger.debug("Token-level errors:\n" + token_cm.summary())
logger.info("Accuracy: %.2f", metrics[1])
logger.info("Error: %.2f", metrics[0])
return metrics[0], metrics[1]
def run_epoch(self, sess, train, labels, dev=None, dev_labels=None):
prog = Progbar(target=1 + int(len(train) / self.config.batch_size))
for i, (enc_batch, dec_batch) in enumerate(
minibatches(train, labels, self.config.batch_size)):
loss = self.train_on_batch(sess, enc_batch, dec_batch)
prog.update(i + 1, [("train loss", loss)])
logger.info("Finished Epoch! Running Dev tests")
if dev != None and dev_labels != None:
return self.dev_loss(sess, dev, dev_labels)
return loss
def output(self, sess, inputs_raw, inputs=None):
if inputs is None:
inputs = self.preprocess_sequence_data(self.helper.vectorize(inputs_raw))
preds = []
prog = Progbar(target=1 + int(len(inputs) / self.config.batch_size))
for i, batch in enumerate(minibatches(inputs, self.config.batch_size, shuffle=False)):
batch = batch[:1] + batch[2:]
preds_ = self.predict_on_batch(sess, *batch)
preds += list(preds_)
prog.update(i + 1, [])
return self.consolidate_predictions(inputs_raw, inputs, preds)
def run_epoch(self, sess, train, labels):
prog = Progbar(target=1 + int(len(train) / self.config.batch_size))
losses, grad_norms = [], []
for i, (enc_batch, dec_batch) in enumerate(
minibatches(train, labels, self.config.batch_size)):
#print batch
loss = self.train_on_batch(sess, enc_batch, dec_batch)
#losses.append(loss)
#grad_norms.append(grad_norm)
#loss = 1
prog.update(i + 1, [("train loss", loss)])
return loss
def dev_loss(self, sess, dev_processed, labels_processed):
prog = Progbar(target=1 +
int(len(dev_processed) / self.config.batch_size))
tot_loss = 0
for i, (enc_batch, dec_batch) in enumerate(
minibatches(dev_processed,
labels_processed,
self.config.batch_size,
shuffle=False)):
feed = self.create_feed_dict(enc_batch, labels_batch=dec_batch)
_, loss = sess.run([self.test_op, self.predict_loss],
feed_dict=feed)
tot_loss += loss
prog.update(i + 1, [("dev loss", tot_loss)])
return tot_loss
def output(self, sess, inputs_raw, inputs):
"""
Reports the output of the model on examples (uses helper to featurize each example).
"""
# always require valid inputs arg
# if inputs is None:
# inputs = self.preprocess_sequence_data(self.helper.vectorize(inputs_raw))
preds = []
prog = Progbar(target=1 + int(len(inputs) / self.config.batch_size))
for i, batch in enumerate(
minibatches(inputs, self.config.batch_size, shuffle=False)):
# Ignore predict
batch = batch[:2] + batch[3:]
preds_ = self.predict_on_batch(sess, *batch)
preds += list(preds_)
prog.update(i + 1, [])
return self.consolidate_predictions(inputs_raw, inputs, preds)
def run_epoch(self, sess, train_examples, dev_set, train_examples_raw,
dev_set_raw):
prog = Progbar(target=1 +
int(len(train_examples) / self.config.batch_size))
for i, batch in enumerate(
minibatches(train_examples, self.config.batch_size)):
loss = self.train_on_batch(sess, *batch)
prog.update(i + 1, [("train loss", loss)])
if self.report: self.report.log_train_loss(loss)
print("")
logger.info("Evaluating on development data")
token_cm, entity_scores = self.evaluate(sess, dev_set, dev_set_raw)
logger.debug("Token-level confusion matrix:\n" + token_cm.as_table())
logger.debug("Token-level scores:\n" + token_cm.summary())
logger.info("Entity level P/R/F1: %.2f/%.2f/%.2f", *entity_scores)
f1 = entity_scores[-1]
return f1
def run_epoch(self, sess, train_examples, dev_set, train_examples_raw, dev_set_raw, epoch):
prog = Progbar(target=1 + int(len(train_examples) / self.config.batch_size))
curr_loss = 0.
num_encountered = 0
for i, batch in enumerate(minibatches(train_examples, self.config.batch_size)):
loss = self.train_on_batch(sess, *batch)
prog.update(i + 1, [("train loss", loss)])
curr_loss += loss
num_encountered += 1
if self.report: self.report.log_train_loss(loss)
train_loss.append(curr_loss/num_encountered)
print("")
logger.info("Evaluating on development data")
divergence = self.evaluate(sess, dev_set, dev_set_raw)
logger.info("KL divergence: %.2f", divergence)
dev_loss.append(divergence)
return divergence
def run_epoch(self, sess, train_examples, dev_set, logfile=None):
prog = Progbar(target=1 +
train_examples[0].shape[0] / self.config.batch_size)
for i, (inputs_batch, outputs_batch) in enumerate(
minibatches(train_examples, self.config.batch_size)):
loss = self.train_on_batch(sess,
inputs_batch,
outputs_batch,
get_loss=True)
prog.update(i + 1, [("train loss", loss)])
print("")
print("Evaluating on train set...")
train_loss = self.eval_batches(sess, train_examples,
self.config.n_eval_batches)
print("Train Loss: {0:.6f}".format(train_loss))
print("Evaluating on dev set...")
dev_loss = self.eval_batches(sess, dev_set, self.config.n_eval_batches)
print("Dev Loss: {0:.6f}".format(dev_loss))
logfile.write(",{0:.5f},{1:.5f}\n".format(float(train_loss),
float(dev_loss)))
return dev_loss
def predict(self, sess, dev):
predicted_sentences = []
dev_processed, labels_processed = self.preprocess_sequence_data(
dev[0], dev[1])
prog = Progbar(target=int(len(dev_processed) / self.config.batch_size))
# predict in batches
tot_loss = 0
for i, (enc_batch, dec_batch) in enumerate(
minibatches(dev_processed, labels_processed,
self.config.batch_size, False)):
#print enc_batch.shape, dec_batch.shape
feed = self.create_feed_dict(enc_batch, dec_batch)
if self.beam_search != None:
outputs = sess.run(self.test_op, feed_dict=feed)
else:
outputs, loss = sess.run([self.test_op, self.predict_loss],
feed_dict=feed)
#loss = 0
#outputs = sess.run(self.test_op, feed_dict = feed)
tot_loss += loss
for j in range(outputs.shape[0]):
predicted = self.construct_sentence(
outputs[j, :], dev[2][(i * self.config.batch_size) + j])
predicted_sentences.append(predicted)
#print predicted
if self.beam_search != None:
prog.update(i + 1)
else:
prog.update(i + 1, [("test loss", tot_loss / (i + 1))])
print ""
return predicted_sentences
def train(model, optimizer, epochs, criterion, train_loader, config, post_process, validate_loader,output_path):
for epoch_index in range(epochs):
model.train()
pbar = Progbar(target=len(train_loader))
index_train = epoch_index * len(train_loader)
train_loss = 0.0
P, R, F1 = 0, 0, 0
for batch_index, batch in enumerate(train_loader):
batch_index_ = batch_index
batch_index_ += index_train
# lr = optimizer.param_groups[0]['lr']
lr = ajust_learning_tri(optimizer, batch_index_, step_size=len(train_loader) * 8)
# 数据进行转换和丢到gpu
for key, value in batch.items():
if value is not None:
if isinstance(value, torch.Tensor):
batch[key] = value.cuda()
preds = model(batch['img'])
loss_dict = criterion(preds, batch)
# backward
optimizer.zero_grad()
loss_dict['loss'].backward()
optimizer.step()
precision, recall, f1 = get_f1_score(preds[:, 0, :, :], batch['shrink_map'], batch['shrink_mask'],
config['post_processing']['args']['thresh'])
train_loss += loss_dict['loss'].item()
P += precision
R += recall
F1 += f1
pbar.update(batch_index + 1, values=[('loss', train_loss / (batch_index + 1)),
('P', P / (batch_index + 1)),
('R', R / (batch_index + 1)),
('F1', F1 / (batch_index + 1)),
('epoch:', epoch_index)])
lr_list.append(lr)
eval(model, optimizer, criterion, post_process, validate_loader, output_path)
def run_epoch(self,
session,
epoch_num,
training_set,
vocab,
validation_set,
sample_size=400):
set_num = len(training_set)
batch_size = self.config.batch_size
batch_num = int(np.ceil(set_num * 1.0 / batch_size))
sample_size = 400
prog = Progbar(target=batch_num)
avg_loss = 0
for i, batch in enumerate(
minibatches(training_set,
self.config.batch_size,
shuffle=False)):
global_batch_num = batch_num * epoch_num + i
_, summary, loss = self.optimize(session, batch)
prog.update(i + 1, [("training loss", loss)])
if self.config.tensorboard and global_batch_num % self.config.log_batch_num == 0:
self.train_writer.add_summary(summary, global_batch_num)
if (i + 1) % self.config.log_batch_num == 0:
logging.info('')
self.evaluate_answer(session,
training_set,
vocab,
sample=sample_size,
log=True)
self.evaluate_answer(session,
validation_set,
vocab,
sample=sample_size,
log=True)
avg_loss += loss
avg_loss /= batch_num
logging.info("Average training loss: {}".format(avg_loss))
return avg_loss
def predict(self, sess, dev, labels):
predicted_sentences = []
dev_processed, labels_processed = self.preprocess_sequence_data(
dev, labels)
prog = Progbar(target=1 +
int(len(dev_processed) / self.config.batch_size))
# predict in batches
for i, (enc_batch, dec_batch) in enumerate(
minibatches(dev_processed,
labels_processed,
self.config.batch_size,
shuffle=False)):
#print enc_batch.shape, dec_batch.shape
feed = self.create_feed_dict(enc_batch, dec_batch)
outputs = sess.run(self.test_op, feed_dict=feed)
for j in range(outputs.shape[0]):
predicted = self.construct_sentence(outputs[j, :])
#print predicted
predicted_sentences.append(predicted)
prog.update(i + 1)
return predicted_sentences
def validate(self, sess, valid_dataset):
"""
Iterate through the validation dataset and determine what
the validation cost is.
This method calls self.test() which explicitly calculates validation cost.
How you implement this function is dependent on how you design
your data iteration function
:return:
"""
batch_num = int(
np.ceil(len(valid_dataset) * 1.0 / self.config.batch_size))
prog = Progbar(target=batch_num)
avg_loss = 0
for i, batch in enumerate(
minibatches(valid_dataset, self.config.batch_size)):
loss = self.test(sess, batch)[0]
prog.update(i + 1, [("validation loss", loss)])
avg_loss += loss
avg_loss /= batch_num
logging.info("Average validation loss: {}".format(avg_loss))
return avg_loss
def train(student_model, teacher_model, optimizer, epochs, student_criterion,
teacher_criterion, train_loader, config, post_process,
validate_loader, output_path):
for epoch_index in range(epochs):
student_model.train()
pbar = Progbar(target=len(train_loader))
index_train = epoch_index * len(train_loader)
train_loss = 0.0
P, R, F1 = 0, 0, 0
for batch_index, batch in enumerate(train_loader):
batch_index_ = batch_index
batch_index_ += index_train
# lr = optimizer.param_groups[0]['lr']
lr = ajust_learning_tri(optimizer,
batch_index_,
step_size=len(train_loader) * 8)
# 数据进行转换和丢到gpu
for key, value in batch.items():
if value is not None:
if isinstance(value, torch.Tensor):
batch[key] = value.cuda()
student_preds = student_model(batch['img'])
loss_dict = {}
# KD loss
if teacher_model is not None:
teacher_model.eval()
with torch.no_grad():
# (b,2,h,w)
teacher_outputs = teacher_model(
batch['img']
) # shrink_maps, threshold_maps, binary_maps
kd_loss_dict = teacher_criterion(student_preds,
teacher_outputs, batch)
loss_dict = {**kd_loss_dict}
gt_loss_dict = student_criterion(student_preds, batch)
loss_dict = {**gt_loss_dict, **loss_dict}
# backward
total_losses = sum(loss_ for loss_ in loss_dict.values())
optimizer.zero_grad()
total_losses.backward()
optimizer.step()
precision, recall, f1 = get_f1_score(
student_preds[:, 0, :, :], batch['shrink_map'],
batch['shrink_mask'],
config['post_processing']['args']['thresh'])
train_loss += total_losses.item()
P += precision
R += recall
F1 += f1
pbar.update(batch_index + 1,
values=[('loss', train_loss / (batch_index + 1)),
('P', P / (batch_index + 1)),
('R', R / (batch_index + 1)),
('F1', F1 / (batch_index + 1)),
('epoch:', epoch_index)])
lr_list.append(lr)
if (epoch_index + 1) % 10 == 0:
eval(student_model, optimizer, post_process, validate_loader,
output_path)