def evaluate(model, eval_dataset, aug_eval=False, scales=1.0, flip_horizontal=True, flip_vertical=False, is_slide=False, stride=None, crop_size=None, num_workers=0, print_detail=True): """ Launch evalution. Args: model(nn.Layer): A sementic segmentation model. eval_dataset (paddle.io.Dataset): Used to read and process validation datasets. aug_eval (bool, optional): Whether to use mulit-scales and flip augment for evaluation. Default: False. scales (list|float, optional): Scales for augment. It is valid when `aug_eval` is True. Default: 1.0. flip_horizontal (bool, optional): Whether to use flip horizontally augment. It is valid when `aug_eval` is True. Default: True. flip_vertical (bool, optional): Whether to use flip vertically augment. It is valid when `aug_eval` is True. Default: False. is_slide (bool, optional): Whether to evaluate by sliding window. Default: False. stride (tuple|list, optional): The stride of sliding window, the first is width and the second is height. It should be provided when `is_slide` is True. crop_size (tuple|list, optional): The crop size of sliding window, the first is width and the second is height. It should be provided when `is_slide` is True. num_workers (int, optional): Num workers for data loader. Default: 0. print_detail (bool, optional): Whether to print detailed information about the evaluation process. Default: True. Returns: float: The mIoU of validation datasets. float: The accuracy of validation datasets. """ model.eval() nranks = paddle.distributed.ParallelEnv().nranks local_rank = paddle.distributed.ParallelEnv().local_rank if nranks > 1: # Initialize parallel environment if not done. if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized( ): paddle.distributed.init_parallel_env() batch_sampler = paddle.io.DistributedBatchSampler(eval_dataset, batch_size=1, shuffle=False, drop_last=False) loader = paddle.io.DataLoader( eval_dataset, batch_sampler=batch_sampler, num_workers=num_workers, return_list=True, ) total_iters = len(loader) intersect_area_all = 0 pred_area_all = 0 label_area_all = 0 if print_detail: logger.info( "Start evaluating (total_samples={}, total_iters={})...".format( len(eval_dataset), total_iters)) progbar_val = progbar.Progbar(target=total_iters, verbose=1) reader_cost_averager = TimeAverager() batch_cost_averager = TimeAverager() batch_start = time.time() with paddle.no_grad(): for iter, (im, label) in enumerate(loader): reader_cost_averager.record(time.time() - batch_start) label = label.astype('int64') ori_shape = label.shape[-2:] if aug_eval: pred = infer.aug_inference( model, im, ori_shape=ori_shape, transforms=eval_dataset.transforms.transforms, scales=scales, flip_horizontal=flip_horizontal, flip_vertical=flip_vertical, is_slide=is_slide, stride=stride, crop_size=crop_size) else: pred = infer.inference( model, im, ori_shape=ori_shape, transforms=eval_dataset.transforms.transforms, is_slide=is_slide, stride=stride, crop_size=crop_size) intersect_area, pred_area, label_area = metrics.calculate_area( pred, label, eval_dataset.num_classes, ignore_index=eval_dataset.ignore_index) # Gather from all ranks if nranks > 1: intersect_area_list = [] pred_area_list = [] label_area_list = [] paddle.distributed.all_gather(intersect_area_list, intersect_area) paddle.distributed.all_gather(pred_area_list, pred_area) paddle.distributed.all_gather(label_area_list, label_area) # Some image has been evaluated and should be eliminated in last iter if (iter + 1) * nranks > len(eval_dataset): valid = len(eval_dataset) - iter * nranks intersect_area_list = intersect_area_list[:valid] pred_area_list = pred_area_list[:valid] label_area_list = label_area_list[:valid] for i in range(len(intersect_area_list)): intersect_area_all = intersect_area_all + intersect_area_list[ i] pred_area_all = pred_area_all + pred_area_list[i] label_area_all = label_area_all + label_area_list[i] else: intersect_area_all = intersect_area_all + intersect_area pred_area_all = pred_area_all + pred_area label_area_all = label_area_all + label_area batch_cost_averager.record(time.time() - batch_start, num_samples=len(label)) batch_cost = batch_cost_averager.get_average() reader_cost = reader_cost_averager.get_average() if local_rank == 0 and print_detail: progbar_val.update(iter + 1, [('batch_cost', batch_cost), ('reader cost', reader_cost)]) reader_cost_averager.reset() batch_cost_averager.reset() batch_start = time.time() class_iou, miou = metrics.mean_iou(intersect_area_all, pred_area_all, label_area_all) class_acc, acc = metrics.accuracy(intersect_area_all, pred_area_all) kappa = metrics.kappa(intersect_area_all, pred_area_all, label_area_all) if print_detail: logger.info( "[EVAL] #Images={} mIoU={:.4f} Acc={:.4f} Kappa={:.4f} ".format( len(eval_dataset), miou, acc, kappa)) logger.info("[EVAL] Class IoU: \n" + str(np.round(class_iou, 4))) logger.info("[EVAL] Class Acc: \n" + str(np.round(class_acc, 4))) return miou, acc
def predict(model, model_path, transforms, image_list, image_dir=None, save_dir='output', aug_pred=False, scales=1.0, flip_horizontal=True, flip_vertical=False, is_slide=False, stride=None, crop_size=None): """ predict and visualize the image_list. Args: model (nn.Layer): Used to predict for input image. model_path (str): The path of pretrained model. transforms (transform.Compose): Preprocess for input image. image_list (list): A list of images to be predicted. image_dir (str): The directory of the images to be predicted. Default: None. save_dir (str): The directory to save the visualized results. Default: 'output'. """ para_state_dict = paddle.load(model_path) model.set_dict(para_state_dict) model.eval() added_saved_dir = os.path.join(save_dir, 'added_prediction') pred_saved_dir = os.path.join(save_dir, 'pseudo_color_prediction') logger.info("Start to predict...") progbar_pred = progbar.Progbar(target=len(image_list), verbose=1) for i, im_path in enumerate(image_list): im = cv2.imread(im_path) ori_shape = im.shape[:2] im, _ = transforms(im) im = im[np.newaxis, ...] im = paddle.to_tensor(im) if aug_pred: pred = infer.aug_inference(model, im, ori_shape=ori_shape, transforms=transforms.transforms, scales=scales, flip_horizontal=flip_horizontal, flip_vertical=flip_vertical, is_slide=is_slide, stride=stride, crop_size=crop_size) else: pred = infer.inference(model, im, ori_shape=ori_shape, transforms=transforms.transforms, is_slide=is_slide, stride=stride, crop_size=crop_size) pred = paddle.squeeze(pred) pred = pred.numpy().astype('uint8') # get the saved name if image_dir is not None: im_file = im_path.replace(image_dir, '') else: im_file = os.path.basename(im_path) if im_file[0] == '/': im_file = im_file[1:] # save added image added_image = utils.visualize.visualize(im_path, pred, weight=0.6) added_image_path = os.path.join(added_saved_dir, im_file) mkdir(added_image_path) cv2.imwrite(added_image_path, added_image) # save pseudo color prediction pred_mask = utils.visualize.get_pseudo_color_map(pred) pred_saved_path = os.path.join(pred_saved_dir, im_file.rsplit(".")[0] + ".png") mkdir(pred_saved_path) pred_mask.save(pred_saved_path) # pred_im = utils.visualize(im_path, pred, weight=0.0) # pred_saved_path = os.path.join(pred_saved_dir, im_file) # mkdir(pred_saved_path) # cv2.imwrite(pred_saved_path, pred_im) progbar_pred.update(i + 1)
def predict(model, model_path, transforms, image_list, image_dir=None, save_dir='output', aug_pred=False, scales=1.0, flip_horizontal=True, flip_vertical=False, is_slide=False, stride=None, crop_size=None, custom_color=None): """ predict and visualize the image_list. Args: model (nn.Layer): Used to predict for input image. model_path (str): The path of pretrained model. transforms (transform.Compose): Preprocess for input image. image_list (list): A list of image path to be predicted. image_dir (str, optional): The root directory of the images predicted. Default: None. save_dir (str, optional): The directory to save the visualized results. Default: 'output'. aug_pred (bool, optional): Whether to use mulit-scales and flip augment for predition. Default: False. scales (list|float, optional): Scales for augment. It is valid when `aug_pred` is True. Default: 1.0. flip_horizontal (bool, optional): Whether to use flip horizontally augment. It is valid when `aug_pred` is True. Default: True. flip_vertical (bool, optional): Whether to use flip vertically augment. It is valid when `aug_pred` is True. Default: False. is_slide (bool, optional): Whether to predict by sliding window. Default: False. stride (tuple|list, optional): The stride of sliding window, the first is width and the second is height. It should be provided when `is_slide` is True. crop_size (tuple|list, optional): The crop size of sliding window, the first is width and the second is height. It should be provided when `is_slide` is True. custom_color (list, optional): Save images with a custom color map. Default: None, use paddleseg's default color map. """ utils.utils.load_entire_model(model, model_path) model.eval() nranks = paddle.distributed.get_world_size() local_rank = paddle.distributed.get_rank() if nranks > 1: img_lists = partition_list(image_list, nranks) else: img_lists = [image_list] added_saved_dir = os.path.join(save_dir, 'added_prediction') pred_saved_dir = os.path.join(save_dir, 'pseudo_color_prediction') logger.info("Start to predict...") progbar_pred = progbar.Progbar(target=len(img_lists[0]), verbose=1) color_map = visualize.get_color_map_list(256, custom_color=custom_color) with paddle.no_grad(): for i, im_path in enumerate(img_lists[local_rank]): data = preprocess(im_path, transforms) if aug_pred: pred, _ = infer.aug_inference(model, data['img'], trans_info=data['trans_info'], scales=scales, flip_horizontal=flip_horizontal, flip_vertical=flip_vertical, is_slide=is_slide, stride=stride, crop_size=crop_size) else: pred, _ = infer.inference(model, data['img'], trans_info=data['trans_info'], is_slide=is_slide, stride=stride, crop_size=crop_size) pred = paddle.squeeze(pred) pred = pred.numpy().astype('uint8') # get the saved name if image_dir is not None: im_file = im_path.replace(image_dir, '') else: im_file = os.path.basename(im_path) if im_file[0] == '/' or im_file[0] == '\\': im_file = im_file[1:] # save added image added_image = utils.visualize.visualize(im_path, pred, color_map, weight=0.6) added_image_path = os.path.join(added_saved_dir, im_file) mkdir(added_image_path) cv2.imwrite(added_image_path, added_image) # save pseudo color prediction pred_mask = utils.visualize.get_pseudo_color_map(pred, color_map) pred_saved_path = os.path.join( pred_saved_dir, os.path.splitext(im_file)[0] + ".png") mkdir(pred_saved_path) pred_mask.save(pred_saved_path) progbar_pred.update(i + 1)
def evaluate(model, eval_dataset, aug_eval=False, scales=1.0, flip_horizontal=True, flip_vertical=False, is_slide=False, stride=None, crop_size=None, num_workers=0): model.eval() nranks = paddle.distributed.ParallelEnv().nranks local_rank = paddle.distributed.ParallelEnv().local_rank if nranks > 1: # Initialize parallel environment if not done. if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized( ): paddle.distributed.init_parallel_env() batch_sampler = paddle.io.DistributedBatchSampler( eval_dataset, batch_size=1, shuffle=False, drop_last=False) loader = paddle.io.DataLoader( eval_dataset, batch_sampler=batch_sampler, num_workers=num_workers, return_list=True, ) total_iters = len(loader) intersect_area_all = 0 pred_area_all = 0 label_area_all = 0 logger.info("Start evaluating (total_samples={}, total_iters={})...".format( len(eval_dataset), total_iters)) progbar_val = progbar.Progbar(target=total_iters, verbose=1) timer = Timer() for iter, (im, label) in enumerate(loader): reader_cost = timer.elapsed_time() label = label.astype('int64') ori_shape = label.shape[-2:] if aug_eval: pred = infer.aug_inference( model, im, ori_shape=ori_shape, transforms=eval_dataset.transforms.transforms, scales=scales, flip_horizontal=flip_horizontal, flip_vertical=flip_vertical, is_slide=is_slide, stride=stride, crop_size=crop_size) else: pred = infer.inference( model, im, ori_shape=ori_shape, transforms=eval_dataset.transforms.transforms, is_slide=is_slide, stride=stride, crop_size=crop_size) intersect_area, pred_area, label_area = metrics.calculate_area( pred, label, eval_dataset.num_classes, ignore_index=eval_dataset.ignore_index) # Gather from all ranks if nranks > 1: intersect_area_list = [] pred_area_list = [] label_area_list = [] paddle.distributed.all_gather(intersect_area_list, intersect_area) paddle.distributed.all_gather(pred_area_list, pred_area) paddle.distributed.all_gather(label_area_list, label_area) # Some image has been evaluated and should be eliminated in last iter if (iter + 1) * nranks > len(eval_dataset): valid = len(eval_dataset) - iter * nranks intersect_area_list = intersect_area_list[:valid] pred_area_list = pred_area_list[:valid] label_area_list = label_area_list[:valid] for i in range(len(intersect_area_list)): intersect_area_all = intersect_area_all + intersect_area_list[i] pred_area_all = pred_area_all + pred_area_list[i] label_area_all = label_area_all + label_area_list[i] else: intersect_area_all = intersect_area_all + intersect_area pred_area_all = pred_area_all + pred_area label_area_all = label_area_all + label_area batch_cost = timer.elapsed_time() timer.restart() if local_rank == 0: progbar_val.update(iter + 1, [('batch_cost', batch_cost), ('reader cost', reader_cost)]) class_iou, miou = metrics.mean_iou(intersect_area_all, pred_area_all, label_area_all) class_acc, acc = metrics.accuracy(intersect_area_all, pred_area_all) kappa = metrics.kappa(intersect_area_all, pred_area_all, label_area_all) logger.info("[EVAL] #Images={} mIoU={:.4f} Acc={:.4f} Kappa={:.4f} ".format( len(eval_dataset), miou, acc, kappa)) logger.info("[EVAL] Class IoU: \n" + str(np.round(class_iou, 4))) logger.info("[EVAL] Class Acc: \n" + str(np.round(class_acc, 4))) return miou, acc