def new_eval_end_event(self, run_states):
"""
Paddlehub default handler for eval_end_event, it will complete visualization and metrics calculation
Args:
run_states (object): the results in eval phase
"""
eval_scores, eval_loss, run_speed = self._calculate_metrics(run_states)
if 'train' in self._envs:
self.vdl_writer.add_scalar(tag="Loss_{}".format(self.phase),
value=eval_loss,
step=self._envs['train'].current_step)
log_scores = ""
s = []
for metric in eval_scores:
if 'train' in self._envs:
self.vdl_writer.add_scalar(
tag="{}_{}".format(metric, self.phase),
value=eval_scores[metric],
step=self._envs['train'].current_step)
# dev[metric].add_record(self._envs['train'].current_step, eval_scores[metric])
log_scores += "%s=%.5f " % (metric, eval_scores[metric])
s.append(eval_scores[metric])
# dev[metric].add_record(self.current_step,eval_scores[metric])
logger.eval(
"[%s dataset evaluation result] loss=%.5f %s[step/sec: %.2f]" %
(self.phase, eval_loss, log_scores, run_speed))
s.append(eval_loss)
if 'train' in self._envs:
s = [self._envs['train'].current_step] + s
# dev_loss.add_record(self._envs['train'].current_step,eval_loss)
s = [str(x) for x in s]
with open('./work/log/%s_dev%s.txt' % (args.do_model, id),
'a',
encoding='utf-8') as f:
f.write(','.join(s) + '\n')
eval_scores_items = eval_scores.items()
if len(eval_scores_items):
# The first metric will be chose to eval
if (args.dev_goal == 'f1'):
main_metric, main_value = list(eval_scores_items)[0]
else: #loss
main_metric, main_value = "negative loss", -eval_loss
else:
logger.warning(
"None of metrics has been implemented, loss will be used to evaluate."
)
# The larger, the better
main_metric, main_value = "negative loss", -eval_loss
if self.phase in ["dev", "val"] and main_value > self.best_score:
self.best_score = main_value
model_saved_dir = os.path.join(self.config.checkpoint_dir,
"best_model")
logger.eval("best model saved to %s [best %s=%.5f]" %
(model_saved_dir, main_metric, main_value))
self.save_inference_model(dirname=model_saved_dir)
def _default_eval_end_event(self, run_states):
"""
Paddlehub default handler for eval_end_event, it will complete visualization and metrics calculation
Args:
run_states (object): the results in eval phase
"""
eval_scores, eval_loss, run_speed = self._calculate_metrics(run_states)
if 'train' in self._envs:
self.tb_writer.add_scalar(
tag="Loss_{}".format(self.phase),
scalar_value=eval_loss,
global_step=self._envs['train'].current_step)
log_scores = ""
for metric in eval_scores:
if 'train' in self._envs:
self.tb_writer.add_scalar(
tag="{}_{}".format(metric, self.phase),
scalar_value=eval_scores[metric],
global_step=self._envs['train'].current_step)
log_scores += "%s=%.5f " % (metric, eval_scores[metric])
logger.eval(
"[%s dataset evaluation result] loss=%.5f %s[step/sec: %.2f]" %
(self.phase, eval_loss, log_scores, run_speed))
eval_scores_items = eval_scores.items()
if len(eval_scores_items):
# The first metric will be chose to eval
main_metric, main_value = list(eval_scores_items)[0]
else:
logger.warning(
"None of metrics has been implemented, loss will be used to evaluate."
)
# The larger, the better
main_metric, main_value = "negative loss", -eval_loss
if self.phase in ["dev", "val"] and main_value > self.best_score:
self.best_score = main_value
model_saved_dir = os.path.join(self.config.checkpoint_dir,
"best_model")
logger.eval("best model saved to %s [best %s=%.5f]" %
(model_saved_dir, main_metric, main_value))
self.save_inference_model(dirname=model_saved_dir)