def interpretation_predict(model, images): images = images.astype('float32') arrange_transforms( model.model_type, model.__class__.__name__, transforms=model.test_transforms, mode='test') tmp_transforms = copy.deepcopy(model.test_transforms.transforms) model.test_transforms.transforms = model.test_transforms.transforms[-2:] new_imgs = [] for i in range(images.shape[0]): images[i] = cv2.cvtColor(images[i], cv2.COLOR_RGB2BGR) new_imgs.append(model.test_transforms(images[i])[0]) new_imgs = np.array(new_imgs) with fluid.scope_guard(model.scope): out = model.exe.run( model.test_prog, feed={'image': new_imgs}, fetch_list=list(model.interpretation_feats.values())) model.test_transforms.transforms = tmp_transforms return out
def export_quant_model(self, dataset, save_dir, batch_size=1, batch_num=10, cache_dir="./temp"): input_channel = getattr(self, 'input_channel', 3) arrange_transforms( model_type=self.model_type, class_name=self.__class__.__name__, transforms=dataset.transforms, mode='quant', input_channel=input_channel) dataset.num_samples = batch_size * batch_num import paddle version = paddle.__version__.strip().split('.') if version[0] == '2' or (version[0] == '0' and hasattr(paddle, 'enable_static')): from .slim.post_quantization import PaddleXPostTrainingQuantizationV2 as PaddleXPostTrainingQuantization else: from .slim.post_quantization import PaddleXPostTrainingQuantization PaddleXPostTrainingQuantization._collect_target_varnames is_use_cache_file = True if cache_dir is None: is_use_cache_file = False quant_prog = self.test_prog.clone(for_test=True) post_training_quantization = PaddleXPostTrainingQuantization( executor=self.exe, dataset=dataset, program=quant_prog, inputs=self.test_inputs, outputs=self.test_outputs, batch_size=batch_size, batch_nums=batch_num, scope=self.scope, algo='KL', quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"], is_full_quantize=False, is_use_cache_file=is_use_cache_file, cache_dir=cache_dir) post_training_quantization.quantize() post_training_quantization.save_quantized_model(save_dir) model_info = self.get_model_info() model_info['status'] = 'Quant' # 保存模型输出的变量描述 model_info['_ModelInputsOutputs'] = dict() model_info['_ModelInputsOutputs']['test_inputs'] = [ [k, v.name] for k, v in self.test_inputs.items() ] model_info['_ModelInputsOutputs']['test_outputs'] = [ [k, v.name] for k, v in self.test_outputs.items() ] with open( osp.join(save_dir, 'model.yml'), encoding='utf-8', mode='w') as f: yaml.dump(model_info, f)
def normlime(img_file, model, dataset=None, num_samples=3000, batch_size=50, save_dir='./', normlime_weights_file=None): """使用NormLIME算法将模型预测结果的可解释性可视化。 NormLIME是利用一定数量的样本来出一个全局的解释。由于NormLIME计算量较大,此处采用一种简化的方式: 使用一定数量的测试样本(目前默认使用所有测试样本),对每个样本进行特征提取,映射到同一个特征空间; 然后以此特征做为输入,以模型输出做为输出,使用线性回归对其进行拟合,得到一个全局的输入和输出的关系。 之后,对一测试样本进行解释时,使用NormLIME全局的解释,来对LIME的结果进行滤波,使最终的可视化结果更加稳定。 注意1:dataset读取的是一个数据集,该数据集不宜过大,否则计算时间会较长,但应包含所有类别的数据。 注意2:NormLIME可解释性结果可视化目前只支持分类模型。 Args: img_file (str): 预测图像路径。 model (paddlex.cv.models): paddlex中的模型。 dataset (paddlex.datasets): 数据集读取器,默认为None。 num_samples (int): LIME用于学习线性模型的采样数,默认为3000。 batch_size (int): 预测数据batch大小,默认为50。 save_dir (str): 可解释性可视化结果(保存为png格式文件)和中间文件存储路径。 normlime_weights_file (str): NormLIME初始化文件名,若不存在,则计算一次,保存于该路径;若存在,则直接载入。 """ assert model.model_type == 'classifier', \ 'Now the interpretation visualize only be supported in classifier!' if model.status != 'Normal': raise Exception( 'The interpretation only can deal with the Normal model') if not osp.exists(save_dir): os.makedirs(save_dir) arrange_transforms(model.model_type, model.__class__.__name__, transforms=model.test_transforms, mode='test') tmp_transforms = copy.deepcopy(model.test_transforms) tmp_transforms.transforms = tmp_transforms.transforms[:-2] img = tmp_transforms(img_file)[0] img = np.around(img).astype('uint8') img = np.expand_dims(img, axis=0) interpreter = None if dataset is None: raise Exception( 'The dataset is None. Cannot implement this kind of interpretation' ) interpreter = get_normlime_interpreter( img, model, dataset, num_samples=num_samples, batch_size=batch_size, save_dir=save_dir, normlime_weights_file=normlime_weights_file) img_name = osp.splitext(osp.split(img_file)[-1])[0] interpreter.interpret(img, save_dir=osp.join(save_dir, img_name))
def _preprocess(images, transforms, model_type, class_name, thread_pool=None): arrange_transforms(model_type=model_type, class_name=class_name, transforms=transforms, mode='test') if thread_pool is not None: batch_data = thread_pool.map(transforms, images) else: batch_data = list() for image in images: batch_data.append(transforms(image)) padding_batch = generate_minibatch(batch_data) im = np.array([data[0] for data in padding_batch]) return im
def lime(img_file, model, num_samples=3000, batch_size=50, save_dir='./'): """使用LIME算法将模型预测结果的可解释性可视化。 LIME表示与模型无关的局部可解释性,可以解释任何模型。LIME的思想是以输入样本为中心, 在其附近的空间中进行随机采样,每个采样通过原模型得到新的输出,这样得到一系列的输入 和对应的输出,LIME用一个简单的、可解释的模型(比如线性回归模型)来拟合这个映射关系, 得到每个输入维度的权重,以此来解释模型。 注意:LIME可解释性结果可视化目前只支持分类模型。 Args: img_file (str): 预测图像路径。 model (paddlex.cv.models): paddlex中的模型。 num_samples (int): LIME用于学习线性模型的采样数,默认为3000。 batch_size (int): 预测数据batch大小,默认为50。 save_dir (str): 可解释性可视化结果(保存为png格式文件)和中间文件存储路径。 """ assert model.model_type == 'classifier', \ 'Now the interpretation visualize only be supported in classifier!' if model.status != 'Normal': raise Exception( 'The interpretation only can deal with the Normal model') if not osp.exists(save_dir): os.makedirs(save_dir) arrange_transforms(model.model_type, model.__class__.__name__, transforms=model.test_transforms, mode='test') tmp_transforms = copy.deepcopy(model.test_transforms) tmp_transforms.transforms = tmp_transforms.transforms[:-2] img = tmp_transforms(img_file)[0] img = np.around(img).astype('uint8') img = np.expand_dims(img, axis=0) interpreter = None interpreter = get_lime_interpreter(img, model, num_samples=num_samples, batch_size=batch_size) img_name = osp.splitext(osp.split(img_file)[-1])[0] interpreter.interpret(img, save_dir=osp.join(save_dir, img_name))
def _preprocess(images, transforms, model_type, class_name, thread_pool=None, input_channel=3): arrange_transforms(model_type=model_type, class_name=class_name, transforms=transforms, mode='test', input_channel=input_channel) if thread_pool is not None: batch_data = thread_pool.map(transforms, images) else: batch_data = list() for image in images: batch_data.append(transforms(image)) padding_batch = generate_minibatch(batch_data) im = np.array([data[0] for data in padding_batch], dtype=padding_batch[0][0].dtype) im_info = [data[1] for data in padding_batch] return im, im_info
def evaluate(self, eval_dataset, batch_size=1, epoch_id=None, metric=None, return_details=False): """评估。 Args: eval_dataset (paddlex.datasets): 验证数据读取器。 batch_size (int): 验证数据批大小。默认为1。当前只支持设置为1。 epoch_id (int): 当前评估模型所在的训练轮数。 metric (bool): 训练过程中评估的方式,取值范围为['COCO', 'VOC']。默认为None, 根据用户传入的Dataset自动选择,如为VOCDetection,则metric为'VOC'; 如为COCODetection,则metric为'COCO'。 return_details (bool): 是否返回详细信息。默认值为False。 Returns: tuple (metrics, eval_details) /dict (metrics): 当return_details为True时,返回(metrics, eval_details), 当return_details为False时,返回metrics。metrics为dict,包含关键字:'bbox_mmap'和'segm_mmap' 或者’bbox_map‘和'segm_map',分别表示预测框和分割区域平均准确率平均值在 各个IoU阈值下的结果取平均值的结果(mmAP)、平均准确率平均值(mAP)。eval_details为dict, 包含bbox、mask和gt三个关键字。其中关键字bbox的键值是一个列表,列表中每个元素代表一个预测结果, 一个预测结果是一个由图像id,预测框类别id, 预测框坐标,预测框得分组成的列表。 关键字mask的键值是一个列表,列表中每个元素代表各预测框内物体的分割结果,分割结果由图像id、 预测框类别id、表示预测框内各像素点是否属于物体的二值图、预测框得分。 而关键字gt的键值是真实标注框的相关信息。 """ input_channel = getattr(self, 'input_channel', 3) arrange_transforms(model_type=self.model_type, class_name=self.__class__.__name__, transforms=eval_dataset.transforms, mode='eval', input_channel=input_channel) if metric is None: if hasattr(self, 'metric') and self.metric is not None: metric = self.metric else: if isinstance(eval_dataset, paddlex.datasets.CocoDetection): metric = 'COCO' else: raise Exception( "eval_dataset should be datasets.COCODetection.") assert metric in ['COCO', 'VOC'], "Metric only support 'VOC' or 'COCO'" if batch_size > 1: batch_size = 1 logging.warning( "Mask RCNN supports batch_size=1 only during evaluating, so batch_size is forced to be set to 1." ) data_generator = eval_dataset.generator(batch_size=batch_size, drop_last=False) total_steps = math.ceil(eval_dataset.num_samples * 1.0 / batch_size) results = list() logging.info( "Start to evaluating(total_samples={}, total_steps={})...".format( eval_dataset.num_samples, total_steps)) for step, data in tqdm.tqdm(enumerate(data_generator()), total=total_steps): images = np.array([d[0] for d in data]).astype('float32') im_infos = np.array([d[1] for d in data]).astype('float32') im_shapes = np.array([d[3] for d in data]).astype('float32') feed_data = { 'image': images, 'im_info': im_infos, 'im_shape': im_shapes, } with fluid.scope_guard(self.scope): outputs = self.exe.run(self.test_prog, feed=[feed_data], fetch_list=list( self.test_outputs.values()), return_numpy=False) res = { 'bbox': (np.array(outputs[0]), outputs[0].recursive_sequence_lengths()), 'mask': (np.array(outputs[1]), outputs[1].recursive_sequence_lengths()) } res_im_id = [d[2] for d in data] res['im_info'] = (im_infos, []) res['im_shape'] = (im_shapes, []) res['im_id'] = (np.array(res_im_id), []) results.append(res) logging.debug("[EVAL] Epoch={}, Step={}/{}".format( epoch_id, step + 1, total_steps)) ap_stats, eval_details = eval_results( results, 'COCO', eval_dataset.coco_gt, with_background=True, resolution=self.mask_head_resolution) if metric == 'VOC': if isinstance(ap_stats[0], np.ndarray) and isinstance( ap_stats[1], np.ndarray): metrics = OrderedDict( zip(['bbox_map', 'segm_map'], [ap_stats[0][1], ap_stats[1][1]])) else: metrics = OrderedDict(zip(['bbox_map', 'segm_map'], [0.0, 0.0])) elif metric == 'COCO': if isinstance(ap_stats[0], np.ndarray) and isinstance( ap_stats[1], np.ndarray): metrics = OrderedDict( zip(['bbox_mmap', 'segm_mmap'], [ap_stats[0][0], ap_stats[1][0]])) else: metrics = OrderedDict( zip(['bbox_mmap', 'segm_mmap'], [0.0, 0.0])) if return_details: return metrics, eval_details return metrics
def evaluate(self, eval_dataset, batch_size=1, epoch_id=None, return_details=False): """评估。 Args: eval_dataset (paddlex.datasets): 验证数据读取器。 batch_size (int): 验证数据批大小。默认为1。 epoch_id (int): 当前评估模型所在的训练轮数。 return_details (bool): 是否返回详细信息。 Returns: dict: 当return_details为False时,返回dict, 包含关键字:'acc1'、'acc5', 分别表示最大值的accuracy、前5个最大值的accuracy。 tuple (metrics, eval_details): 当return_details为True时,增加返回dict, 包含关键字:'true_labels'、'pred_scores',分别代表真实类别id、每个类别的预测得分。 """ arrange_transforms(model_type=self.model_type, class_name=self.__class__.__name__, transforms=eval_dataset.transforms, mode='eval') data_generator = eval_dataset.generator(batch_size=batch_size, drop_last=False) k = min(5, self.num_classes) total_steps = math.ceil(eval_dataset.num_samples * 1.0 / batch_size) true_labels = list() pred_scores = list() if not hasattr(self, 'parallel_test_prog'): with fluid.scope_guard(self.scope): self.parallel_test_prog = fluid.CompiledProgram( self.test_prog).with_data_parallel( share_vars_from=self.parallel_train_prog) batch_size_each_gpu = self._get_single_card_bs(batch_size) logging.info( "Start to evaluating(total_samples={}, total_steps={})...".format( eval_dataset.num_samples, total_steps)) for step, data in tqdm.tqdm(enumerate(data_generator()), total=total_steps): images = np.array([d[0] for d in data]).astype('float32') labels = [d[1] for d in data] num_samples = images.shape[0] if num_samples < batch_size: num_pad_samples = batch_size - num_samples pad_images = np.tile(images[0:1], (num_pad_samples, 1, 1, 1)) images = np.concatenate([images, pad_images]) with fluid.scope_guard(self.scope): outputs = self.exe.run(self.parallel_test_prog, feed={'image': images}, fetch_list=list( self.test_outputs.values())) outputs = [outputs[0][:num_samples]] true_labels.extend(labels) pred_scores.extend(outputs[0].tolist()) logging.debug("[EVAL] Epoch={}, Step={}/{}".format( epoch_id, step + 1, total_steps)) pred_top1_label = np.argsort(pred_scores)[:, -1] pred_topk_label = np.argsort(pred_scores)[:, -k:] acc1 = sum(pred_top1_label == true_labels) / len(true_labels) acck = sum( [np.isin(x, y) for x, y in zip(true_labels, pred_topk_label)]) / len(true_labels) metrics = OrderedDict([('acc1', acc1), ('acc{}'.format(k), acck)]) if return_details: eval_details = { 'true_labels': true_labels, 'pred_scores': pred_scores } return metrics, eval_details return metrics
def evaluate(self, eval_dataset, batch_size=1, epoch_id=None, return_details=False): """评估。 Args: eval_dataset (paddlex.datasets): 评估数据读取器。 batch_size (int): 评估时的batch大小。默认1。 epoch_id (int): 当前评估模型所在的训练轮数。 return_details (bool): 是否返回详细信息。默认False。 Returns: dict: 当return_details为False时,返回dict。包含关键字:'miou'、'category_iou'、'macc'、 'category_acc'和'kappa',分别表示平均iou、各类别iou、平均准确率、各类别准确率和kappa系数。 tuple (metrics, eval_details):当return_details为True时,增加返回dict (eval_details), 包含关键字:'confusion_matrix',表示评估的混淆矩阵。 """ arrange_transforms(model_type=self.model_type, class_name=self.__class__.__name__, transforms=eval_dataset.transforms, mode='eval') total_steps = math.ceil(eval_dataset.num_samples * 1.0 / batch_size) conf_mat = ConfusionMatrix(self.num_classes, streaming=True) data_generator = eval_dataset.generator(batch_size=batch_size, drop_last=False) if not hasattr(self, 'parallel_test_prog'): with fluid.scope_guard(self.scope): self.parallel_test_prog = fluid.CompiledProgram( self.test_prog).with_data_parallel( share_vars_from=self.parallel_train_prog) logging.info( "Start to evaluating(total_samples={}, total_steps={})...".format( eval_dataset.num_samples, total_steps)) for step, data in tqdm.tqdm(enumerate(data_generator()), total=total_steps): images = np.array([d[0] for d in data]) im_info = [d[1] for d in data] labels = [d[2] for d in data] num_samples = images.shape[0] if num_samples < batch_size: num_pad_samples = batch_size - num_samples pad_images = np.tile(images[0:1], (num_pad_samples, 1, 1, 1)) images = np.concatenate([images, pad_images]) feed_data = {'image': images} with fluid.scope_guard(self.scope): outputs = self.exe.run(self.parallel_test_prog, feed=feed_data, fetch_list=list( self.test_outputs.values()), return_numpy=True) pred = outputs[0] if num_samples < batch_size: pred = pred[0:num_samples] for i in range(num_samples): one_pred = np.squeeze(pred[i]).astype('uint8') one_label = labels[i] for info in im_info[i][::-1]: if info[0] == 'resize': w, h = info[1][1], info[1][0] one_pred = cv2.resize(one_pred, (w, h), cv2.INTER_NEAREST) elif info[0] == 'padding': w, h = info[1][1], info[1][0] one_pred = one_pred[0:h, 0:w] one_pred = one_pred.astype('int64') one_pred = one_pred[np.newaxis, :, :, np.newaxis] one_label = one_label[np.newaxis, np.newaxis, :, :] mask = one_label != self.ignore_index conf_mat.calculate(pred=one_pred, label=one_label, ignore=mask) _, iou = conf_mat.mean_iou() logging.debug("[EVAL] Epoch={}, Step={}/{}, iou={}".format( epoch_id, step + 1, total_steps, iou)) category_iou, miou = conf_mat.mean_iou() category_acc, oacc = conf_mat.accuracy() category_f1score = conf_mat.f1_score() metrics = OrderedDict( zip([ 'miou', 'category_iou', 'oacc', 'category_acc', 'kappa', 'category_F1-score' ], [ miou, category_iou, oacc, category_acc, conf_mat.kappa(), category_f1score ])) if return_details: eval_details = { 'confusion_matrix': conf_mat.confusion_matrix.tolist() } return metrics, eval_details return metrics
def train_loop(self, num_epochs, train_dataset, train_batch_size, eval_dataset=None, save_interval_epochs=1, log_interval_steps=10, save_dir='output', use_vdl=False, early_stop=False, early_stop_patience=5): if train_dataset.num_samples < train_batch_size: raise Exception( 'The amount of training datset must be larger than batch size.') if not osp.isdir(save_dir): if osp.exists(save_dir): os.remove(save_dir) os.makedirs(save_dir) if use_vdl: from visualdl import LogWriter vdl_logdir = osp.join(save_dir, 'vdl_log') # 给transform添加arrange操作 input_channel = getattr(self, 'input_channel', 3) arrange_transforms( model_type=self.model_type, class_name=self.__class__.__name__, transforms=train_dataset.transforms, mode='train', input_channel=input_channel) # 构建train_data_loader self.build_train_data_loader( dataset=train_dataset, batch_size=train_batch_size) if eval_dataset is not None: self.eval_transforms = eval_dataset.transforms self.test_transforms = copy.deepcopy(eval_dataset.transforms) # 获取实时变化的learning rate lr = self.optimizer._learning_rate if isinstance(lr, fluid.framework.Variable): self.train_outputs['lr'] = lr # 在多卡上跑训练 if self.parallel_train_prog is None: build_strategy = fluid.compiler.BuildStrategy() build_strategy.fuse_all_optimizer_ops = False if paddlex.env_info['place'] != 'cpu' and len(self.places) > 1: build_strategy.sync_batch_norm = self.sync_bn exec_strategy = fluid.ExecutionStrategy() exec_strategy.num_iteration_per_drop_scope = 1 self.parallel_train_prog = fluid.CompiledProgram( self.train_prog).with_data_parallel( loss_name=self.train_outputs['loss'].name, build_strategy=build_strategy, exec_strategy=exec_strategy) total_num_steps = math.floor(train_dataset.num_samples / train_batch_size) num_steps = 0 time_stat = list() time_train_one_epoch = None time_eval_one_epoch = None total_num_steps_eval = 0 # 模型总共的评估次数 total_eval_times = math.ceil(num_epochs / save_interval_epochs) # 检测目前仅支持单卡评估,训练数据batch大小与显卡数量之商为验证数据batch大小。 eval_batch_size = train_batch_size if self.model_type == 'detector': eval_batch_size = self._get_single_card_bs(train_batch_size) if eval_dataset is not None: total_num_steps_eval = math.ceil(eval_dataset.num_samples / eval_batch_size) if use_vdl: # VisualDL component log_writer = LogWriter(vdl_logdir) thresh = 0.0001 if early_stop: earlystop = EarlyStop(early_stop_patience, thresh) best_accuracy_key = "" best_accuracy = -1.0 best_model_epoch = -1 start_epoch = self.completed_epochs # task_id: 目前由PaddleX GUI赋值 # 用于在VisualDL日志中注明所属任务id task_id = getattr(paddlex, "task_id", "") for i in range(start_epoch, num_epochs): records = list() step_start_time = time.time() epoch_start_time = time.time() for step, data in enumerate(self.train_data_loader()): outputs = self.exe.run( self.parallel_train_prog, feed=data, fetch_list=list(self.train_outputs.values())) outputs_avg = np.mean(np.array(outputs), axis=1) records.append(outputs_avg) # 训练完成剩余时间预估 current_time = time.time() step_cost_time = current_time - step_start_time step_start_time = current_time if len(time_stat) < 20: time_stat.append(step_cost_time) else: time_stat[num_steps % 20] = step_cost_time # 每间隔log_interval_steps,输出loss信息 num_steps += 1 if num_steps % log_interval_steps == 0: step_metrics = OrderedDict( zip(list(self.train_outputs.keys()), outputs_avg)) if use_vdl: for k, v in step_metrics.items(): log_writer.add_scalar( '{}-Metrics/Training(Step): {}'.format( task_id, k), v, num_steps) # 估算剩余时间 avg_step_time = np.mean(time_stat) if time_train_one_epoch is not None: eta = (num_epochs - i - 1) * time_train_one_epoch + ( total_num_steps - step - 1) * avg_step_time else: eta = ((num_epochs - i) * total_num_steps - step - 1 ) * avg_step_time if time_eval_one_epoch is not None: eval_eta = ( total_eval_times - i // save_interval_epochs ) * time_eval_one_epoch else: eval_eta = ( total_eval_times - i // save_interval_epochs ) * total_num_steps_eval * avg_step_time eta_str = seconds_to_hms(eta + eval_eta) logging.info( "[TRAIN] Epoch={}/{}, Step={}/{}, {}, time_each_step={}s, eta={}" .format(i + 1, num_epochs, step + 1, total_num_steps, dict2str(step_metrics), round(avg_step_time, 2), eta_str)) train_metrics = OrderedDict( zip(list(self.train_outputs.keys()), np.mean( records, axis=0))) logging.info('[TRAIN] Epoch {} finished, {} .'.format( i + 1, dict2str(train_metrics))) time_train_one_epoch = time.time() - epoch_start_time epoch_start_time = time.time() # 每间隔save_interval_epochs, 在验证集上评估和对模型进行保存 self.completed_epochs += 1 eval_epoch_start_time = time.time() if (i + 1) % save_interval_epochs == 0 or i == num_epochs - 1: current_save_dir = osp.join(save_dir, "epoch_{}".format(i + 1)) if not osp.isdir(current_save_dir): os.makedirs(current_save_dir) if getattr(self, 'use_ema', False): self.exe.run(self.ema.apply_program) if eval_dataset is not None and eval_dataset.num_samples > 0: self.eval_metrics, self.eval_details = self.evaluate( eval_dataset=eval_dataset, batch_size=eval_batch_size, epoch_id=i + 1, return_details=True) logging.info('[EVAL] Finished, Epoch={}, {} .'.format( i + 1, dict2str(self.eval_metrics))) # 保存最优模型 best_accuracy_key = list(self.eval_metrics.keys())[0] current_accuracy = self.eval_metrics[best_accuracy_key] if current_accuracy > best_accuracy: best_accuracy = current_accuracy best_model_epoch = i + 1 best_model_dir = osp.join(save_dir, "best_model") self.save_model(save_dir=best_model_dir) if use_vdl: for k, v in self.eval_metrics.items(): if isinstance(v, list): continue if isinstance(v, np.ndarray): if v.size > 1: continue log_writer.add_scalar( "{}-Metrics/Eval(Epoch): {}".format( task_id, k), v, i + 1) self.save_model(save_dir=current_save_dir) if getattr(self, 'use_ema', False): self.exe.run(self.ema.restore_program) time_eval_one_epoch = time.time() - eval_epoch_start_time eval_epoch_start_time = time.time() if best_model_epoch > 0: logging.info( 'Current evaluated best model in eval_dataset is epoch_{}, {}={}' .format(best_model_epoch, best_accuracy_key, best_accuracy)) if eval_dataset is not None and early_stop: if earlystop(current_accuracy): break