def test(self) -> None: dataset_len = len(self._dataloader.dataset) pbar = tqdm(total=dataset_len) with torch.no_grad(): start_time = time.time() for item in self._dataloader: imgs = item['image'].to(self._device) size = imgs.shape[-2:] preds = self._model(imgs) if isinstance(preds, (list, tuple)): preds = preds[0].cpu() else: preds = preds.cpu() preds = self._label_transform_func(preds) for i in range(preds.shape[0]): filename = os.path.splitext( item['filename'][i])[0] + '.png' filepath = pathjoin(self._img_save_path, filename) img = preds[i] # if img.shape[-2:] != size: # img = cv2.resize(img,dsize=(size[1],size[0]),interpolation=cv2.INTER_LINEAR) cv2.imwrite(filepath, img) #pbar.update(1) #pbar.set_description("Processing %s" % filename) print(f"{ dataset_len / (time.time()-start_time) } FPS")
def __call__(self, model_dict, isFinal: bool = False, hooks=False): self.__timer += 1 if self.__timer % self.__save_interval == 0 or isFinal: self.__interval_timer += 1 model_ext_name = 'final' if isFinal else str(self.__interval_timer) model_save_path = pathjoin( self.__save_dir_path, self.__base_model_name + '-' + model_ext_name + '.pth') torch.save(model_dict, model_save_path) if hooks: hooks(model_save_path, self.__interval_timer)
def __init__(self,save_interval,save_dir_path=None,save_base_name:str ='model'): self.__timer = 0 self.__save_interval = save_interval self.__save_dir_path = save_dir_path if self.__save_dir_path is None: self.__save_dir_path = pathjoin( '../', time.strftime("%F %H-%M-%S",time.localtime()) ) mkdirs(self.__save_dir_path) self.__base_model_name = save_base_name self.__interval_timer = 0
def __call__(self, model:Module, isFinal:bool = False,save_base_name = None,is_add=True): if is_add: self.__timer += 1 if self.__timer % self.__save_interval == 0 or isFinal: self.__interval_timer += 1 model_ext_name = 'final' if isFinal else str(self.__interval_timer) if save_base_name is None: save_base_name = self.__base_model_name model_save_path = pathjoin( self.__save_dir_path, save_base_name+'-'+model_ext_name+'.pth' ) torch.save(model.state_dict(),model_save_path)
def test(self) -> None: dataset_len = len(self._dataloader.dataset) pbar = tqdm(total=dataset_len) with torch.no_grad(): for item in self._dataloader: imgs = item['image'].to(self._device) preds = self._model(imgs).cpu() preds = self._label_transform_func(preds) for i in range(preds.shape[0]): filename = os.path.splitext(item['filename'][i])[0]+'.png' filepath = pathjoin(self._img_save_path,filename) cv2.imwrite(filepath, preds[i]) pbar.update(1) pbar.set_description("Processing %s" % filename)
def evaluate(config: Configuration) -> None: if config.DISABLE_EVAL: return if config.USE_GPU is not None: device = torch.device('cuda') else: device = torch.device('cpu') sal_measure = get_measure(config.EVAL_MEASURE_LIST, config.EVALUATOR_DIR, config.DATASET_TEST_GT_DIR, device) measure = OrderedDict() measure['setting'] = config.MODEL_SAVE_DIR_NAME measure['dataset'] = config.DATASET_NAME measure.update(sal_measure) for key, item in measure.items(): if not isinstance(item, Sequence): measure[key] = [item] else: measure[key] = [str(item)] table_content = pandas2markdown(pd.DataFrame(measure)) record_file_dir = os.path.dirname(config.EVALUATOR_SUMMARY_FILE_PATH) mkdirs(record_file_dir) # Write xlsx report xlsx_filename = os.path.splitext( os.path.basename(config.EVALUATOR_SUMMARY_FILE_PATH))[0] + '.xlsx' xlsx_filepath = pathjoin(record_file_dir, xlsx_filename) data_dict = OrderedDict( **{ 'setting': str(config.MODEL_SAVE_DIR_NAME), 'dataset': str(config.DATASET_NAME), 'g_lr': str(config.G_LEARNING_RATE), 'd_lr': str(config.D_LEARNING_RATE), 'step_size': str(config.STEP_INTERVAL), 'batch_size': str(config.BATCH_SIZE), 'crop_size': str(config.CROP_SIZE), 'partial_data': str(config.PARTIAL_DATA), 'max_iter': str(config.MAX_ITER), 'lambda_pred_sal': str(config.LAMBDA_PRED_SAL), 'lambda_pred_adv': str(config.LAMBDA_PRED_ADV), 'lambda_semi_sal': str(config.LAMBDA_SEMI_SAL), 'lambda_semi_adv': str(config.LAMBDA_SEMI_ADV), }) data_dict.update(sal_measure) record_dataframe = pd.DataFrame(data_dict, index=[0]) if not os.path.exists(xlsx_filepath): record_dataframe.to_excel(xlsx_filepath, index=False) else: pd.concat([pd.read_excel(xlsx_filepath), record_dataframe], sort=False).to_excel(xlsx_filepath, index=False) title = config.MODEL_SAVE_DIR_NAME file_content = (f"\n" f"# Experiment {title} \n" f"Time:{time.strftime('%Y-%m-%d %X')} \n" f"Dataset:{config.DATASET_NAME} \n" f"Test folder:{config.EVALUATOR_DIR} \n" f"Test index:{' '.join(config.EVAL_MEASURE_LIST)} \n" f"Command parameters:\n" f"```bash\n" f"{config.CMD_STR}\n" f"```\n" f"\n" f"## Experimental results\n" f"{table_content}\n") with portalocker.Lock(config.EVALUATOR_SUMMARY_FILE_PATH, 'a+', \ encoding='utf-8',timeout=600) as f: f.write(file_content)
def init(self, use_arg_parser=True, *args, **kwargs): super().__init__(*args, **kwargs) # running setting self.DISABLE_TRAIN = False self.DISABLE_TEST = False self.DISABLE_EVAL = False # dataset setting self.DATASET_TRAIN_ROOT_DIR = None self.DATASET_TRAIN_LIST_PATH = None self.DATASET_TEST_ROOT_DIR = None self.DATASET_TEST_LIST_PATH = None self.DATASET_NAME = None self.DATASET_TRAIN_DIR = None self.DATASET_TEST_DIR = None self.DATASET_IS_SCALE = False self.RANDOM_BRIGHT = False # training setting self.BATCH_SIZE = 1 self.CROP_SIZE = None self.LEARNING_RATE = 0.007 self.EPOCH = 20 self.MOMENTUM = 0.9 self.WEIGHT_DECAY = 0.0 self.PRETRAINED_MODEL_PATH = None self.STEP_INTERVAL = 10 self.DROP_RATE = 0 self.DISABLE_VISUAL = False self.OPTIM = "ADAM" self.MODEL_NAME = "efficientnet-b3" self.RESNET_SCALE = 4 # gpu self.USE_GPU = None # model saving self.MODEL_SAVE_ROOT_DIR = None self.MODEL_SAVE_DIR_NAME = None self.LOG_DIR = pathjoin( os.path.dirname(__file__), 'logs', ) # hyper parameters self.TRAIN_DOWN_ITER = 2 / 3 self.S_LOSS_GAMA = 3 # testting model path self.TEST_MODEL_PTH_PATH = None # eval setting self.EVALUATOR_DIR = None self.EVAL_MEASURE_LIST = None self.EVALUATOR_SUMMARY_FILE_PATH = None self.EVALUATOR_DATASETS = None # proc name self.PROC_NAME = None parser = ArgumentParser(description="Configuration of the System") parser.add_argument('-is_scale', '--DATASET_IS_SCALE', action='store_true', \ default=self.DATASET_IS_SCALE) parser.add_argument('-random_bright', '--RANDOM_BRIGHT', action='store_true', \ default=self.RANDOM_BRIGHT) parser.add_argument('-model_name', '--MODEL_NAME', type=str, default=self.MODEL_NAME) parser.add_argument('-s_loss_gama', '--S_LOSS_GAMA', type=int, default=self.S_LOSS_GAMA) parser.add_argument('-resnet_scale', '--RESNET_SCALE', type=float, default=self.RESNET_SCALE) parser.add_argument('-train_down_iter', '--TRAIN_DOWN_ITER', type=float, default=self.TRAIN_DOWN_ITER) parser.add_argument('-disable_test', '--DISABLE_TEST', action='store_true', \ default=self.DISABLE_TEST) parser.add_argument('-disable_train', '--DISABLE_TRAIN', action='store_true', \ default=self.DISABLE_TRAIN) parser.add_argument('-disable_eval', '--DISABLE_EVAL', action='store_true', \ default=self.DISABLE_EVAL) parser.add_argument('-disable_visual', '--DISABLE_VISUAL', action='store_true', \ default=self.DISABLE_VISUAL) parser.add_argument('-d', '--DATASET_NAME', required=True, type=str, \ default=self.DATASET_NAME, \ choices="DUT-OMRON DUTS ECSSD HKU-IS PASCAL-S SOD".split(" ")) parser.add_argument('-b', '--BATCH_SIZE', type=int, default=self.BATCH_SIZE) parser.add_argument('-crop', '--CROP_SIZE', type=int, default=self.CROP_SIZE) parser.add_argument('-lr', '--LEARNING_RATE', type=float, default=self.LEARNING_RATE) parser.add_argument('-epoch', '--EPOCH', type=int, default=self.EPOCH) parser.add_argument('-weight_decay', '--WEIGHT_DECAY', type=float, default=self.WEIGHT_DECAY) parser.add_argument('-drop_rate', '--DROP_RATE', type=float, default=self.DROP_RATE) parser.add_argument('-step', '--STEP_INTERVAL', type=int, default=self.STEP_INTERVAL) parser.add_argument('-optim', '--OPTIM', type=str, default=self.OPTIM) parser.add_argument('-momentum', '--MOMENTUM', type=float, default=self.MOMENTUM) parser.add_argument('-pretrain', '--PRETRAINED_MODEL_PATH', type=str, \ default=self.PRETRAINED_MODEL_PATH) parser.add_argument('-gpu', '--USE_GPU', type=str, default=self.USE_GPU) parser.add_argument('-proc', '--PROC_NAME', type=str, default=self.PROC_NAME) parser.add_argument('-test_model', '--TEST_MODEL_PTH_PATH', type=str, default=self.TEST_MODEL_PTH_PATH) parser.add_argument('-save', '--MODEL_SAVE_DIR_NAME', required=True, type=str, \ default=self.MODEL_SAVE_DIR_NAME) parser.add_argument('-eval_dir', '--EVALUATOR_DIR', type=str, default=self.EVALUATOR_DIR) parser.add_argument('-eval_m_list', '--EVAL_MEASURE_LIST', nargs='+', \ type=str, choices="max-F mean-F MAE S precision recall".split(" "),default=self.EVAL_MEASURE_LIST) parser.add_argument('-eval_file', '--EVALUATOR_SUMMARY_FILE_PATH', \ type=str, default=self.EVALUATOR_SUMMARY_FILE_PATH) parser.add_argument('-eval_d', '--EVALUATOR_DATASETS', nargs='+', \ type=str, choices="ALL DUT-OMRON DUTS ECSSD HKU-IS PASCAL-S SOD".split(" "), \ default=self.EVALUATOR_DATASETS) if use_arg_parser: args = parser.parse_args() self.update(args.__dict__) # MODEL_SAVE_PATH self.MODEL_SAVE_ROOT_DIR = pathjoin( self.LOG_DIR, self.MODEL_SAVE_DIR_NAME, ) self.MODEL_SAVE_PATH = pathjoin( self.MODEL_SAVE_ROOT_DIR, 'save_models', ) if self.BATCH_SIZE != 1: assert self.CROP_SIZE is not None, "CROP_SIZE can't be null, if " \ "BATCH_SIZE != 1, please use -crop to specify the paramter" if self.EVAL_MEASURE_LIST is None: self.EVAL_MEASURE_LIST = ["max-F", "MAE", "S"] tmp_dict = get_dataset_path_by_name(self.DATASET_NAME) self.DATASET_TRAIN_ROOT_DIR = tmp_dict['train_dir_path'] self.DATASET_TRAIN_LIST_PATH = tmp_dict['train_lst_path'] self.DATASET_TEST_ROOT_DIR = tmp_dict['test_dir_path'] self.DATASET_TEST_LIST_PATH = tmp_dict['test_lst_path'] self.DATASET_TRAIN_DIR = tmp_dict['train_dir_name'] self.DATASET_TEST_DIR = tmp_dict['test_dir_name'] self.DATASET_TEST_GT_DIR = pathjoin(self.DATASET_TEST_ROOT_DIR, 'GT') if self.TEST_MODEL_PTH_PATH is None: self.TEST_MODEL_PTH_PATH = pathjoin(self.MODEL_SAVE_PATH, 'model-final.pth') self.TEST_IMG_SAVE_PATH = pathjoin( self.MODEL_SAVE_ROOT_DIR, 'test', self.DATASET_TEST_DIR if len(self.DATASET_TEST_DIR) != 0 else self.DATASET_NAME) self.OPTIM = self.OPTIM.upper() if self.EVALUATOR_DIR is None: self.EVALUATOR_DIR = self.TEST_IMG_SAVE_PATH if self.EVALUATOR_SUMMARY_FILE_PATH is None: self.EVALUATOR_SUMMARY_FILE_PATH = pathjoin( self.LOG_DIR, 'ExperimentalNotes.md') else: self.EVALUATOR_SUMMARY_FILE_PATH = pathjoin( self.LOG_DIR, self.EVALUATOR_SUMMARY_FILE_PATH) self.CMD_STR = "python " + " ".join(sys.argv) if self.EVALUATOR_DATASETS is not None: if "ALL" in self.EVALUATOR_DATASETS: self.EVALUATOR_DATASETS = "DUT-OMRON DUTS ECSSD HKU-IS PASCAL-S SOD".split( ) self.EVALUATOR_GTS = [] self.TEST_IMG_SAVE_PATHS = [] self.DATASET_TEST_ROOT_DIRS = [] self.DATASET_TEST_LIST_PATHS = [] for dataset_name in self.EVALUATOR_DATASETS: dataset_info = get_dataset_path_by_name(dataset_name) self.EVALUATOR_GTS.append( pathjoin(dataset_info['test_dir_path'], 'GT')) dataset_test_dir_name = dataset_info['test_dir_name'] testpath = pathjoin( self.MODEL_SAVE_ROOT_DIR, 'test', dataset_test_dir_name if len(dataset_test_dir_name) != 0 else dataset_name) self.TEST_IMG_SAVE_PATHS.append(testpath) self.DATASET_TEST_ROOT_DIRS.append( dataset_info['test_dir_path']) self.DATASET_TEST_LIST_PATHS.append( dataset_info['test_lst_path']) self.EVALUATOR_DIRS = self.TEST_IMG_SAVE_PATHS if self.EVALUATOR_DIR not in self.EVALUATOR_DIRS: self.EVALUATOR_DIRS.append(self.EVALUATOR_DIR) self.EVALUATOR_GTS.append(self.DATASET_TEST_GT_DIR) self.DATASET_TEST_ROOT_DIRS.append(self.DATASET_TEST_ROOT_DIR) self.DATASET_TEST_LIST_PATHS.append( self.DATASET_TEST_LIST_PATH) self.EVALUATOR_DATASETS.append(self.DATASET_NAME) else: self.EVALUATOR_DIRS = [self.TEST_IMG_SAVE_PATH] self.EVALUATOR_GTS = [self.DATASET_TEST_GT_DIR] self.TEST_IMG_SAVE_PATHS = [self.TEST_IMG_SAVE_PATH] self.DATASET_TEST_ROOT_DIRS = [self.DATASET_TEST_ROOT_DIR] self.DATASET_TEST_LIST_PATHS = [self.DATASET_TEST_LIST_PATH] self.EVALUATOR_DATASETS = [self.DATASET_NAME]
def init(self, use_arg_parser=True, *args, **kwargs): super().__init__(*args, **kwargs) # Run configuration self.DISABLE_TRAIN = False self.DISABLE_TEST = False self.DISABLE_EVAL = False # dataset configuration self.DATASET_TRAIN_ROOT_DIR = None self.DATASET_TRAIN_LIST_PATH = None self.DATASET_TEST_ROOT_DIR = None self.DATASET_TEST_LIST_PATH = None self.DATASET_NAME = None self.DATASET_TRAIN_DIR = None self.DATASET_TEST_DIR = None # Training configuration self.SEED = 2345 self.BATCH_SIZE = 10 self.CROP_SIZE = 321 self.G_LEARNING_RATE = 2.5e-4 self.D_LEARNING_RATE = 1e-4 self.MAX_ITER = 25000 self.MOMENTUM = 0.9 self.PRETRAINED_MODEL_PATH = None self.STEP_INTERVAL = 1 self.WEIGHT_DECAY = 0.0005 self.DISABLE_VISUAL = False self.DATA_IDX_PKL_PATH = None self.SEMI_START = 5000 self.MASK_T = 0.2 self.PARTIAL_DATA = 0.125 self.LAMBDA_SEMI_ADV = 0.007 self.LAMBDA_SEMI_SAL = 1 self.LAMBDA_PRED_SAL = 1 self.LAMBDA_PRED_ADV = 0.01 self.USE_GRAB = False # Graphics card use configuration self.USE_GPU = None # Model save settings self.MODEL_SAVE_ROOT_DIR = None self.MODEL_SAVE_DIR_NAME = None self.LOG_DIR = pathjoin( os.path.dirname(__file__), 'logs', ) self.DATA_IDX_PKL_SAVE_PATH = pathjoin( self.LOG_DIR, 'idx.pkl', ) # test setting self.TEST_MODEL_PTH_PATH = None # eval setting self.EVALUATOR_DIR = None self.EVAL_MEASURE_LIST = None self.EVALUATOR_SUMMARY_FILE_PATH = None self.EVALUATOR_DATASETS = None # proc_name setting self.PROC_NAME = None parser = ArgumentParser(description="Configuration of the System") parser.add_argument('-disable_test', '--DISABLE_TEST', action='store_true', \ default=self.DISABLE_TEST) parser.add_argument('-disable_train', '--DISABLE_TRAIN', action='store_true', \ default=self.DISABLE_TRAIN) parser.add_argument('-disable_eval', '--DISABLE_EVAL', action='store_true', \ default=self.DISABLE_EVAL) parser.add_argument('-disable_visual', '--DISABLE_VISUAL', action='store_true', \ default=self.DISABLE_VISUAL) parser.add_argument('-seed', '--SEED', type=int, default=self.SEED) parser.add_argument('-d', '--DATASET_NAME', required=True, type=str, \ default=self.DATASET_NAME, \ choices="DUT-OMRON DUTS PASCAL-S SOD".split(" ")) parser.add_argument('-b', '--BATCH_SIZE', type=int, default=self.BATCH_SIZE) parser.add_argument('-crop', '--CROP_SIZE', type=int, default=self.CROP_SIZE) parser.add_argument('-g_lr', '--G_LEARNING_RATE', type=float, default=self.G_LEARNING_RATE) parser.add_argument('-d_lr', '--D_LEARNING_RATE', type=float, default=self.D_LEARNING_RATE) parser.add_argument('-max_iter', '--MAX_ITER', type=int, default=self.MAX_ITER) parser.add_argument('-step', '--STEP_INTERVAL', type=int, default=self.STEP_INTERVAL) parser.add_argument('-momentum', '--MOMENTUM', type=float, default=self.MOMENTUM) parser.add_argument('-weight_decay', '--WEIGHT_DECAY', type=float, default=self.WEIGHT_DECAY) parser.add_argument('-idx', '--DATA_IDX_PKL_PATH', type=str, default=self.DATA_IDX_PKL_PATH) parser.add_argument('-start', '--SEMI_START', type=int, default=self.SEMI_START) parser.add_argument('-mask_T', '--MASK_T', type=float, default=self.MASK_T) parser.add_argument('-part', '--PARTIAL_DATA', type=float, default=self.PARTIAL_DATA) parser.add_argument('-grab', '--USE_GRAB', type=bool, default=self.USE_GRAB) parser.add_argument('-l_semi_adv', '--LAMBDA_SEMI_ADV', type=float, default=self.LAMBDA_SEMI_ADV) parser.add_argument('-l_semi_sal', '--LAMBDA_SEMI_SAL', type=float, default=self.LAMBDA_SEMI_SAL) parser.add_argument('-l_pred_adv', '--LAMBDA_PRED_ADV', type=float, default=self.LAMBDA_PRED_ADV) parser.add_argument('-l_pred_sal', '--LAMBDA_PRED_SAL', type=float, default=self.LAMBDA_PRED_SAL) parser.add_argument('-pretrain', '--PRETRAINED_MODEL_PATH', type=str, \ default=self.PRETRAINED_MODEL_PATH) parser.add_argument('-gpu', '--USE_GPU', type=str, default=self.USE_GPU) parser.add_argument('-proc', '--PROC_NAME', type=str, default=self.PROC_NAME) parser.add_argument('-test_model', '--TEST_MODEL_PTH_PATH', type=str, default=self.TEST_MODEL_PTH_PATH) parser.add_argument('-save', '--MODEL_SAVE_DIR_NAME', required=True, type=str, \ default=self.MODEL_SAVE_DIR_NAME) parser.add_argument('-eval_dir', '--EVALUATOR_DIR', type=str, default=self.EVALUATOR_DIR) parser.add_argument('-eval_m_list', '--EVAL_MEASURE_LIST', nargs='+', \ type=str, choices="max-F mean-F MAE S precision recall".split(" "),default=self.EVAL_MEASURE_LIST) parser.add_argument('-eval_file', '--EVALUATOR_SUMMARY_FILE_PATH', \ type=str, default=self.EVALUATOR_SUMMARY_FILE_PATH) parser.add_argument('-eval_d', '--EVALUATOR_DATASETS', nargs='+', \ type=str, choices="ALL DUT-OMRON DUTS PASCAL-S SOD".split(" "), \ default=self.EVALUATOR_DATASETS) if use_arg_parser: args = parser.parse_args() self.update(args.__dict__) # handle MODEL_SAVE_PATH self.MODEL_SAVE_ROOT_DIR = pathjoin( self.LOG_DIR, self.MODEL_SAVE_DIR_NAME, ) self.MODEL_SAVE_PATH = pathjoin( self.MODEL_SAVE_ROOT_DIR, 'save_models', ) if self.BATCH_SIZE != 1: assert self.CROP_SIZE is not None, "CROP_SIZE can't be null, if " \ "BATCH_SIZE != 1, please use -crop to specify the paramter" # handle EVAL_MEASURE_LIST if self.EVAL_MEASURE_LIST is None: self.EVAL_MEASURE_LIST = ["max-F", "MAE", "S"] # handle DATASET_ROOT_DIR DATASET_LIST_PATH tmp_dict = get_dataset_path_by_name(self.DATASET_NAME) self.DATASET_TRAIN_ROOT_DIR = tmp_dict['train_dir_path'] self.DATASET_TRAIN_LIST_PATH = tmp_dict['train_lst_path'] self.DATASET_TEST_ROOT_DIR = tmp_dict['test_dir_path'] self.DATASET_TEST_LIST_PATH = tmp_dict['test_lst_path'] self.DATASET_TRAIN_DIR = tmp_dict['train_dir_name'] self.DATASET_TEST_DIR = tmp_dict['test_dir_name'] self.DATASET_TEST_GT_DIR = pathjoin(self.DATASET_TEST_ROOT_DIR, 'GT') if self.TEST_MODEL_PTH_PATH is None: self.TEST_MODEL_PTH_PATH = pathjoin(self.MODEL_SAVE_PATH, 'G-final.pth') self.TEST_IMG_SAVE_PATH = pathjoin( self.MODEL_SAVE_ROOT_DIR, 'test', self.DATASET_TEST_DIR if len(self.DATASET_TEST_DIR) != 0 else self.DATASET_NAME) # handle EVALUATOR_DIR if self.EVALUATOR_DIR is None: self.EVALUATOR_DIR = self.TEST_IMG_SAVE_PATH if self.EVALUATOR_SUMMARY_FILE_PATH is None: self.EVALUATOR_SUMMARY_FILE_PATH = pathjoin( self.LOG_DIR, 'ExperimentalNotes.md') else: self.EVALUATOR_SUMMARY_FILE_PATH = pathjoin( self.LOG_DIR, self.EVALUATOR_SUMMARY_FILE_PATH) if self.EVALUATOR_DATASETS is not None: if "ALL" in self.EVALUATOR_DATASETS: self.EVALUATOR_DATASETS = "DUT-OMRON DUTS PASCAL-S SOD".split() self.EVALUATOR_GTS = [] self.TEST_IMG_SAVE_PATHS = [] self.DATASET_TEST_ROOT_DIRS = [] self.DATASET_TEST_LIST_PATHS = [] for dataset_name in self.EVALUATOR_DATASETS: dataset_info = get_dataset_path_by_name(dataset_name) self.EVALUATOR_GTS.append( pathjoin(dataset_info['test_dir_path'], 'GT')) dataset_test_dir_name = dataset_info['test_dir_name'] testpath = pathjoin( self.MODEL_SAVE_ROOT_DIR, 'test', dataset_test_dir_name if len(dataset_test_dir_name) != 0 else dataset_name) self.TEST_IMG_SAVE_PATHS.append(testpath) self.DATASET_TEST_ROOT_DIRS.append( dataset_info['test_dir_path']) self.DATASET_TEST_LIST_PATHS.append( dataset_info['test_lst_path']) self.EVALUATOR_DIRS = self.TEST_IMG_SAVE_PATHS if self.EVALUATOR_DIR not in self.EVALUATOR_DIRS: self.EVALUATOR_DIRS.append(self.EVALUATOR_DIR) self.EVALUATOR_GTS.append(self.DATASET_TEST_GT_DIR) self.DATASET_TEST_ROOT_DIRS.append(self.DATASET_TEST_ROOT_DIR) self.DATASET_TEST_LIST_PATHS.append( self.DATASET_TEST_LIST_PATH) self.EVALUATOR_DATASETS.append(self.DATASET_NAME) else: self.EVALUATOR_DIRS = [self.TEST_IMG_SAVE_PATH] self.EVALUATOR_GTS = [self.DATASET_TEST_GT_DIR] self.TEST_IMG_SAVE_PATHS = [self.TEST_IMG_SAVE_PATH] self.DATASET_TEST_ROOT_DIRS = [self.DATASET_TEST_ROOT_DIR] self.DATASET_TEST_LIST_PATHS = [self.DATASET_TEST_LIST_PATH] self.EVALUATOR_DATASETS = [self.DATASET_NAME] import sys # Command parameter acquisition self.CMD_STR = "python " + " ".join(sys.argv)
def __init__( self, catch_time_interval: int ) -> None: super(MyVisualHelper, self).__init__(catch_time_interval) from torch.utils.tensorboard import SummaryWriter self.writer = SummaryWriter(pathjoin(config.MODEL_SAVE_ROOT_DIR,'running'))
def evaluate(config: Configuration) -> None: if config.DISABLE_EVAL: return if config.USE_GPU is not None: device = torch.device('cuda') else: device = torch.device('cpu') sal_measure = get_measure(config.EVAL_MEASURE_LIST, config.EVALUATOR_DIR,config.DATASET_TEST_GT_DIR, device) assert 'MAE' in sal_measure.keys() measure = OrderedDict() measure['setting'] = config.MODEL_SAVE_DIR_NAME measure['dataset'] = config.DATASET_NAME measure.update(sal_measure) for key,item in measure.items(): if not isinstance(item, Sequence): measure[key] = [item] else: measure[key] = [str(item)] table_content = pandas2markdown(pd.DataFrame(measure)) record_file_dir = os.path.dirname(config.EVALUATOR_SUMMARY_FILE_PATH) mkdirs(record_file_dir) # export cvv report csv_filename = os.path.splitext(os.path.basename(config.EVALUATOR_SUMMARY_FILE_PATH))[0] + '.csv' csv_filepath = pathjoin( record_file_dir, csv_filename ) data_dict = OrderedDict(**{ 'setting':str(config.MODEL_SAVE_DIR_NAME), 'dataset':str(config.DATASET_NAME), 'lr':str(config.LEARNING_RATE), 'epoch':str(config.EPOCH), 'step_size':str(config.STEP_INTERVAL), 'optim':str(config.OPTIM), 'batch_size':str(config.BATCH_SIZE), 'crop_size':str(config.CROP_SIZE), 'weight_decay':str(config.WEIGHT_DECAY), 'drop_rate':str(config.DROP_RATE), }) data_dict.update(sal_measure) record_dataframe = pd.DataFrame(data_dict,index=[0]) if not os.path.exists(csv_filepath): record_dataframe.to_csv(csv_filepath,index=False) else: pd.concat([pd.read_csv(csv_filepath), record_dataframe],sort=False).to_csv(xlsx_filepath,index=False) title = config.MODEL_SAVE_DIR_NAME file_content = (f"\n" f"# setting {title} \n" f"time:{time.strftime('%Y-%m-%d %X')} \n" f"dataset:{config.DATASET_NAME} \n" f"test dir:{config.EVALUATOR_DIR} \n" f"command string:\n" f"```bash\n" f"{config.CMD_STR}\n" f"```\n" f"\n" f"## result\n" f"{table_content}\n") with portalocker.Lock(config.EVALUATOR_SUMMARY_FILE_PATH, 'a+', \ encoding='utf-8',timeout=600) as f: f.write(file_content)