def _create_train_callbacks(self) -> List[Callback]: # TODO: do we want to use early stopping? if so, use the right chechpoint manager and set the correct # `monitor` quantity (example: monitor='val_acc', mode='max') keras_callbacks = [ ModelTrainingStatusTrackerCallback(self.training_status), ModelTrainingProgressLoggerCallback(self.config, self.training_status), ] if self.config.is_saving: keras_callbacks.append( ModelCheckpointSaverCallback(self, self.config.SAVE_EVERY_EPOCHS, self.logger)) # keras_callbacks.append( # ModelCheckpoint(self.config.get_entire_model_path(self.config.MODEL_SAVE_PATH), monitor='val_accuracy', verbose=1, save_best_only=True, # mode='max')) # # save vocabs # model_save_path = self.config.MODEL_SAVE_PATH # model_save_dir = '/'.join(model_save_path.split('/')[:-1]) # if not os.path.isdir(model_save_dir): # os.makedirs(model_save_dir, exist_ok=True) # self.vocabs.save(self.config.get_vocabularies_path_from_model_path(model_save_path)) # # save vocabs end if self.config.is_testing: keras_callbacks.append(ModelEvaluationCallback(self)) if self.config.USE_TENSORBOARD: log_dir = "logs/scalars/train_" + common.now_str() tensorboard_callback = keras.callbacks.TensorBoard( log_dir=log_dir, update_freq=self.config.NUM_BATCHES_TO_LOG_PROGRESS * self.config.TRAIN_BATCH_SIZE) keras_callbacks.append(tensorboard_callback) return keras_callbacks
def _create_train_callbacks(self) -> List[Callback]: # TODO: do we want to use early stopping? if so, use the right chechpoint manager and set the correct # `monitor` quantity (example: monitor='val_acc', mode='max') keras_callbacks = [ ModelTrainingStatusTrackerCallback(self.training_status), ModelTrainingProgressLoggerCallback(self.config, self.training_status), ] if self.config.EARLY_STOPPING: keras_callbacks.append( tf.keras.callbacks.EarlyStopping( monitor='loss', patience=self.config.PATIENCE)) if self.config.is_saving: keras_callbacks.append( ModelCheckpointSaverCallback(self, self.config.SAVE_EVERY_EPOCHS, self.logger)) if self.config.is_testing: keras_callbacks.append(ModelEvaluationCallback(self)) if self.config.USE_TENSORBOARD: log_dir = "logs/scalars/train_" + common.now_str() tensorboard_callback = keras.callbacks.TensorBoard( log_dir=log_dir, update_freq=self.config.NUM_BATCHES_TO_LOG_PROGRESS * self.config.TRAIN_BATCH_SIZE) keras_callbacks.append(tensorboard_callback) return keras_callbacks