def fit(self, train_data, valid_data=None, verbose=True, saved=True, show_progress=False): for epoch_idx in range(self.epochs): self._train_at_once(train_data, valid_data) if (epoch_idx + 1) % self.eval_step == 0: # evaluate valid_start_time = time() valid_score, valid_result = self._valid_epoch(valid_data) self.best_valid_score, self.cur_step, stop_flag, update_flag = early_stopping( valid_score, self.best_valid_score, self.cur_step, max_step=self.stopping_step, bigger=self.valid_metric_bigger) valid_end_time = time() valid_score_output = (set_color("epoch %d evaluating", 'green') + " [" + set_color("time", 'blue') + ": %.2fs, " + set_color("valid_score", 'blue') + ": %f]") % \ (epoch_idx, valid_end_time - valid_start_time, valid_score) valid_result_output = set_color( 'valid result', 'blue') + ': \n' + dict2str(valid_result) if verbose: self.logger.info(valid_score_output) self.logger.info(valid_result_output) self.tensorboard.add_scalar('Vaild_score', valid_score, epoch_idx) if update_flag: if saved: self.model.save_model(self.temp_best_file) self._save_checkpoint(epoch_idx) update_output = set_color( 'Saving current best', 'blue') + ': %s' % self.saved_model_file if verbose: self.logger.info(update_output) self.best_valid_result = valid_result if stop_flag: stop_output = 'Finished training, best eval result in epoch %d' % \ (epoch_idx - self.cur_step * self.eval_step) if self.temp_file: os.remove(self.temp_file) if verbose: self.logger.info(stop_output) break return self.best_valid_score, self.best_valid_result
def finetune(self, train_data, valid_data=None, verbose=True, saved=True): for epoch_idx in range(self.start_epoch, self.epochs): # train training_start_time = time() train_loss = self._train_epoch(train_data, epoch_idx) self.train_loss_dict[epoch_idx] = sum(train_loss) if isinstance( train_loss, tuple) else train_loss training_end_time = time() train_loss_output = \ self._generate_train_loss_output(epoch_idx, training_start_time, training_end_time, train_loss) if verbose: self.logger.info(train_loss_output) # eval if self.eval_step <= 0 or not valid_data: if saved: self._save_checkpoint(epoch_idx) update_output = 'Saving current: %s' % self.saved_model_file if verbose: self.logger.info(update_output) continue if (epoch_idx + 1) % self.eval_step == 0: valid_start_time = time() valid_score, valid_result = self._valid_epoch(valid_data) self.best_valid_score, self.cur_step, stop_flag, update_flag = early_stopping( valid_score, self.best_valid_score, self.cur_step, max_step=self.stopping_step, bigger=self.valid_metric_bigger) valid_end_time = time() valid_score_output = "epoch %d evaluating [time: %.2fs, valid_score: %f]" % \ (epoch_idx, valid_end_time - valid_start_time, valid_score) valid_result_output = 'valid result: \n' + dict2str( valid_result) if verbose: self.logger.info(valid_score_output) self.logger.info(valid_result_output) if update_flag: if saved: self._save_checkpoint(epoch_idx) update_output = 'Saving current best: %s' % self.saved_model_file if verbose: self.logger.info(update_output) self.best_valid_result = valid_result if stop_flag: stop_output = 'Finished training, best eval result in epoch %d' % \ (epoch_idx - self.cur_step * self.eval_step) if verbose: self.logger.info(stop_output) break return self.best_valid_score, self.best_valid_result
def fit(self, train_data, valid_data=None, verbose=True, saved=True): r"""Train the model based on the train data and the valid data. Args: train_data (DataLoader): the train data valid_data (DataLoader, optional): the valid data, default: None. If it's None, the early_stopping is invalid. verbose (bool, optional): whether to write training and evaluation information to logger, default: True saved (bool, optional): whether to save the model parameters, default: True Returns: (float, dict): best valid score and best valid result. If valid_data is None, it returns (-1, None) """ if hasattr(self.model, 'train_preparation'): self.model.train_preparation(train_data=train_data, valid_data=valid_data) for epoch_idx in range(self.start_epoch, self.epochs): # train training_start_time = time() train_loss = self._train_epoch(train_data, epoch_idx) self.train_loss_dict[epoch_idx] = sum(train_loss) if isinstance( train_loss, tuple) else train_loss training_end_time = time() train_loss_output = \ self._generate_train_loss_output(epoch_idx, training_start_time, training_end_time, train_loss) if verbose: self.logger.info(train_loss_output) # eval if self.eval_step <= 0 or not valid_data: if saved: self._save_checkpoint(epoch_idx) update_output = 'Saving current: %s' % self.saved_model_file if verbose: self.logger.info(update_output) continue if (epoch_idx + 1) % self.eval_step == 0: valid_start_time = time() valid_score, valid_result = self._valid_epoch(valid_data) self.best_valid_score, self.cur_step, stop_flag, update_flag = early_stopping( valid_score, self.best_valid_score, self.cur_step, max_step=self.stopping_step, bigger=self.valid_metric_bigger) valid_end_time = time() valid_score_output = "epoch %d evaluating [time: %.2fs, valid_score: %f]" % \ (epoch_idx, valid_end_time - valid_start_time, valid_score) valid_result_output = 'valid result: \n' + dict2str( valid_result) if verbose: self.logger.info(valid_score_output) self.logger.info(valid_result_output) if update_flag: if saved: self._save_checkpoint(epoch_idx) update_output = 'Saving current best: %s' % self.saved_model_file if verbose: self.logger.info(update_output) self.best_valid_result = valid_result if stop_flag: stop_output = 'Finished training, best eval result in epoch %d' % \ (epoch_idx - self.cur_step * self.eval_step) if verbose: self.logger.info(stop_output) break return self.best_valid_score, self.best_valid_result
def fit(self, train_data, valid_data=None, verbose=True, saved=True, show_progress=False, callback_fn=None): r"""Train the model based on the train data and the valid data. Args: train_data (DataLoader): the train data valid_data (DataLoader, optional): the valid data, default: None. If it's None, the early_stopping is invalid. verbose (bool, optional): whether to write training and evaluation information to logger, default: True saved (bool, optional): whether to save the model parameters, default: True show_progress (bool): Show the progress of training epoch and evaluate epoch. Defaults to ``False``. callback_fn (callable): Optional callback function executed at end of epoch. Includes (epoch_idx, valid_score) input arguments. Returns: (float, dict): best valid score and best valid result. If valid_data is None, it returns (-1, None) """ if saved and self.start_epoch >= self.epochs: self._save_checkpoint(-1) self.eval_collector.data_collect(train_data) if 'dynamic' in self.config['train_neg_sample_args'].keys( ) and self.config['train_neg_sample_args']['dynamic'] != 'none': train_data.get_model(self.model) for epoch_idx in range(self.start_epoch, self.epochs): # train training_start_time = time() train_loss = self._train_epoch(train_data, epoch_idx, show_progress=show_progress) self.train_loss_dict[epoch_idx] = sum(train_loss) if isinstance( train_loss, tuple) else train_loss training_end_time = time() train_loss_output = \ self._generate_train_loss_output(epoch_idx, training_start_time, training_end_time, train_loss) if verbose: self.logger.info(train_loss_output) self._add_train_loss_to_tensorboard(epoch_idx, train_loss) # eval if self.eval_step <= 0 or not valid_data: if saved: self._save_checkpoint(epoch_idx) update_output = set_color( 'Saving current', 'blue') + ': %s' % self.saved_model_file if verbose: self.logger.info(update_output) continue if (epoch_idx + 1) % self.eval_step == 0: valid_start_time = time() valid_score, valid_result = self._valid_epoch( valid_data, show_progress=show_progress) self.best_valid_score, self.cur_step, stop_flag, update_flag = early_stopping( valid_score, self.best_valid_score, self.cur_step, max_step=self.stopping_step, bigger=self.valid_metric_bigger) valid_end_time = time() valid_score_output = (set_color("epoch %d evaluating", 'green') + " [" + set_color("time", 'blue') + ": %.2fs, " + set_color("valid_score", 'blue') + ": %f]") % \ (epoch_idx, valid_end_time - valid_start_time, valid_score) valid_result_output = set_color( 'valid result', 'blue') + ': \n' + dict2str(valid_result) if verbose: self.logger.info(valid_score_output) self.logger.info(valid_result_output) self.tensorboard.add_scalar('Vaild_score', valid_score, epoch_idx) if update_flag: if saved: self._save_checkpoint(epoch_idx) update_output = set_color( 'Saving current best', 'blue') + ': %s' % self.saved_model_file if verbose: self.logger.info(update_output) self.best_valid_result = valid_result if callback_fn: callback_fn(epoch_idx, valid_score) if stop_flag: stop_output = 'Finished training, best eval result in epoch %d' % \ (epoch_idx - self.cur_step * self.eval_step) if verbose: self.logger.info(stop_output) break self._add_hparam_to_tensorboard(self.best_valid_score) return self.best_valid_score, self.best_valid_result
def fit(self, train_data, valid_data=None, verbose=True, saved=True, show_progress=False, callback_fn=None): r"""Train the model based on the train data and the valid data. Args: train_data (DataLoader): the train data valid_data (DataLoader, optional): the valid data, default: None. If it's None, the early_stopping is invalid. verbose (bool, optional): whether to write training and evaluation information to logger, default: True saved (bool, optional): whether to save the model parameters, default: True show_progress (bool): Show the progress of training epoch and evaluate epoch. Defaults to ``False``. callback_fn (callable): Optional callback function executed at end of epoch. Includes (epoch_idx, valid_score) input arguments. Returns: (float, dict): best valid score and best valid result. If valid_data is None, it returns (-1, None) """ if saved and self.start_epoch >= self.epochs: self._save_checkpoint(-1) for epoch_idx in range(self.start_epoch, self.epochs): # train training_start_time = time() train_loss = self._train_epoch(train_data, epoch_idx, show_progress=show_progress) self.train_loss_dict[epoch_idx] = sum(train_loss) if isinstance( train_loss, tuple) else train_loss training_end_time = time() train_loss_output = \ self._generate_train_loss_output(epoch_idx, training_start_time, training_end_time, train_loss) #if verbose: # self.logger.info(train_loss_output) # eval if self.eval_step <= 0 or not valid_data: if saved: self._save_checkpoint(epoch_idx) update_output = set_color( 'Saving current', 'blue') + ': %s' % self.saved_model_file #if verbose: # self.logger.info(update_output) continue if (epoch_idx + 1) % self.eval_step == 0: valid_start_time = time() valid_score, valid_result = self._valid_epoch( valid_data, show_progress=show_progress) self.best_valid_score, self.cur_step, stop_flag, update_flag = early_stopping( valid_score, self.best_valid_score, self.cur_step, max_step=self.stopping_step, bigger=self.valid_metric_bigger) valid_end_time = time() valid_score_output = (set_color("epoch %d evaluating", 'green') + " [" + set_color("time", 'blue') + ": %.2fs, " + set_color("valid_score", 'blue') + ": %f]") % \ (epoch_idx, valid_end_time - valid_start_time, valid_score) valid_result_output = set_color( 'valid result', 'blue') + ': \n' + dict2str(valid_result) if verbose: self.logger.info(valid_score_output) names = [k for k, _ in valid_result.items()] values = [round(v, 3) for _, v in valid_result.items()] my_table = PrettyTable() my_table.field_names = names my_table.add_row(values) print(my_table) if update_flag: if saved: self._save_checkpoint(epoch_idx) update_output = set_color( 'Saving current best', 'blue') + ': %s' % self.saved_model_file #if verbose: # self.logger.info(update_output) self.best_valid_result = valid_result if callback_fn: callback_fn(epoch_idx, valid_score) if stop_flag: stop_output = 'Finished training, best eval result in epoch %d' % \ (epoch_idx - self.cur_step * self.eval_step) if verbose: self.logger.info(stop_output) break if self.draw_loss_pic: save_path = '{}-{}-train_loss.pdf'.format(self.config['model'], get_local_time()) self.plot_train_loss(save_path=os.path.join(save_path)) return self.best_valid_score, self.best_valid_result
def fit(self, train_data, valid_data=None, verbose=True, saved=True, show_progress=False, callback_fn=None): if saved and self.start_epoch >= self.epochs: self._save_checkpoint(-1) encoder_params = set(self.model.encoder.parameters()) decoder_params = set(self.model.decoder.parameters()) optimizer_encoder = self._build_optimizer(encoder_params) optimizer_decoder = self._build_optimizer(decoder_params) for epoch_idx in range(self.start_epoch, self.epochs): # alternate training training_start_time = time() train_loss = self._train_epoch(train_data, epoch_idx, show_progress=show_progress, n_epochs=self.n_enc_epochs, encoder_flag=True, optimizer=optimizer_encoder) self.model.update_prior() train_loss = self._train_epoch(train_data, epoch_idx, show_progress=show_progress, n_epochs=self.n_dec_epochs, encoder_flag=False, optimizer=optimizer_decoder) self.train_loss_dict[epoch_idx] = sum(train_loss) if isinstance( train_loss, tuple) else train_loss training_end_time = time() train_loss_output = \ self._generate_train_loss_output(epoch_idx, training_start_time, training_end_time, train_loss) if verbose: self.logger.info(train_loss_output) # eval if self.eval_step <= 0 or not valid_data: if saved: self._save_checkpoint(epoch_idx) update_output = 'Saving current: %s' % self.saved_model_file if verbose: self.logger.info(update_output) continue if (epoch_idx + 1) % self.eval_step == 0: valid_start_time = time() valid_score, valid_result = self._valid_epoch( valid_data, show_progress=show_progress) self.best_valid_score, self.cur_step, stop_flag, update_flag = early_stopping( valid_score, self.best_valid_score, self.cur_step, max_step=self.stopping_step, bigger=self.valid_metric_bigger) valid_end_time = time() valid_score_output = (set_color("epoch %d evaluating", 'green') + " [" + set_color("time", 'blue') + ": %.2fs, " + set_color("valid_score", 'blue') + ": %f]") % \ (epoch_idx, valid_end_time - valid_start_time, valid_score) valid_result_output = set_color( 'valid result', 'blue') + ': \n' + dict2str(valid_result) if verbose: self.logger.info(valid_score_output) self.logger.info(valid_result_output) if update_flag: if saved: self._save_checkpoint(epoch_idx) update_output = set_color( 'Saving current best', 'blue') + ': %s' % self.saved_model_file if verbose: self.logger.info(update_output) self.best_valid_result = valid_result if callback_fn: callback_fn(epoch_idx, valid_score) if stop_flag: stop_output = 'Finished training, best eval result in epoch %d' % \ (epoch_idx - self.cur_step * self.eval_step) if verbose: self.logger.info(stop_output) break return self.best_valid_score, self.best_valid_result
def fit(self, train_data, valid_data=None, verbose=True, saved=True, show_progress=False, callback_fn=None): r"""Train the model based on the train data and the valid data. Args: train_data (DataLoader): the train data valid_data (DataLoader, optional): the valid data, default: None. If it's None, the early_stopping is invalid. verbose (bool, optional): whether to write training and evaluation information to logger, default: True saved (bool, optional): whether to save the model parameters, default: True show_progress (bool): Show the progress of training epoch and evaluate epoch. Defaults to ``False``. callback_fn (callable): Optional callback function executed at end of epoch. Includes (epoch_idx, valid_score) input arguments. Returns: (float, dict): best valid score and best valid result. If valid_data is None, it returns (-1, None) """ if saved and self.start_epoch >= self.epochs: self._save_checkpoint(-1) for epoch_idx in range(self.start_epoch, self.epochs): # train training_start_time = time() train_loss = self._train_epoch(train_data, epoch_idx, show_progress=show_progress) self.train_loss_dict[epoch_idx] = sum(train_loss) if isinstance( train_loss, tuple) else train_loss training_end_time = time() train_loss_output = \ self._generate_train_loss_output(epoch_idx, training_start_time, training_end_time, train_loss) if verbose: self.logger.info(train_loss_output) # eval if self.eval_step <= 0 or not valid_data: if saved: self._save_checkpoint(epoch_idx) update_output = 'Saving current: %s' % self.saved_model_file if verbose: self.logger.info(update_output) continue if (epoch_idx + 1) % self.eval_step == 0: valid_start_time = time() valid_score, valid_result = self._valid_epoch( valid_data, show_progress=show_progress) self.best_valid_score, self.cur_step, stop_flag, update_flag = early_stopping( valid_score, self.best_valid_score, self.cur_step, max_step=self.stopping_step, bigger=self.valid_metric_bigger) valid_end_time = time() valid_score_output = "epoch %d evaluating [time: %.2fs, valid_score: %f]" % \ (epoch_idx, valid_end_time - valid_start_time, valid_score) valid_result_output = 'valid result: \n' + dict2str( valid_result) if verbose: self.logger.info(valid_score_output) self.logger.info(valid_result_output) if update_flag: if saved: self._save_checkpoint(epoch_idx) update_output = 'Saving current best: %s' % self.saved_model_file if verbose: self.logger.info(update_output) self.best_valid_result = valid_result if callback_fn: callback_fn(epoch_idx, valid_score) if stop_flag: stop_output = 'Finished training, best eval result in epoch %d' % \ (epoch_idx - self.cur_step * self.eval_step) if verbose: self.logger.info(stop_output) break if self.draw_pic: self.plot_train_loss(save_path=self.config['model'] + '_train_loss_graph.pdf') return self.best_valid_score, self.best_valid_result