def split_folds(config_file: str): config = load_config(config_file) df = pd.read_csv(config.data.train_df_path) df = df[~df['EncodedPixels'].isnull()] df['file_name'] = df['Image_Label'].map(lambda x: x.split('_')[0]) df['label'] = df['Image_Label'].map(lambda x: x.split('_')[-1]) classes = df['label'].unique() df_g = df.groupby('file_name')['label'].agg(set).reset_index() for class_name in classes: df_g[class_name] = df_g['label'].map( lambda x: 1 if class_name in x else 0) # categorical df_g['categorical'] = df_g.apply( lambda x: str(x.Fish) + str(x.Flower) + str(x.Sugar) + str(x.Gravel) , axis=1) le = preprocessing.LabelEncoder() df_g['categorical_num'] = le.fit_transform(df_g['categorical']) df = pd.merge(df, df_g[['file_name', 'categorical_num']], how='left', on='file_name') kf = StratifiedKFold(n_splits=config.data.params.num_folds) for i, (train_id, val_id) in enumerate( kf.split(df.file_name, df.categorical_num)): train = df.loc[train_id][['file_name']].drop_duplicates() train.to_csv(f'{config.data.fold_dir}/' f'{config.data.fold_train_file}_{i}.csv', index=False) val = df.loc[val_id][['file_name']].drop_duplicates() val.to_csv(f'{config.data.fold_dir}/' f'{config.data.fold_valid_file}_{i}.csv', index=False)
IS_TEST = args.test USE_BOX_MERGE = args.use_box_merge USE_BOX_SCORE = args.use_box_score DATASET_DIR = args.dataset_root_dir if args.dataset_split_file == '': DATASET_SPLIT_FILE = os.path.join(DATASET_DIR, './3DOP_splits/val.txt') else: DATASET_SPLIT_FILE = args.dataset_split_file if args.output_dir == '': OUTPUT_DIR = os.path.join(args.checkpoint_path, './eval/') else: OUTPUT_DIR = args.output_dir CHECKPOINT_PATH = args.checkpoint_path CONFIG_PATH = os.path.join(CHECKPOINT_PATH, 'config') assert os.path.isfile(CONFIG_PATH), 'No config file found in %s' config = load_config(CONFIG_PATH) # setup dataset =============================================================== if IS_TEST: dataset = KittiDataset(os.path.join(DATASET_DIR, 'image/testing/image_2'), os.path.join(DATASET_DIR, 'velodyne/testing/velodyne/'), os.path.join(DATASET_DIR, 'calib/testing/calib/'), '', num_classes=config['num_classes'], is_training=False) else: dataset = KittiDataset(os.path.join(DATASET_DIR, 'image/training/image_2'), os.path.join(DATASET_DIR, 'velodyne/training/velodyne/'), os.path.join(DATASET_DIR, 'calib/training/calib/'), os.path.join(DATASET_DIR,
parser.add_argument('--dataset_split_file', type=str, default='', help='Path to KITTI dataset split file.' 'Default="DATASET_ROOT_DIR/3DOP_splits' '/train_config["train_dataset"]"') args = parser.parse_args() train_config = load_train_config(args.train_config_path) DATASET_DIR = args.dataset_root_dir if args.dataset_split_file == '': DATASET_SPLIT_FILE = os.path.join( DATASET_DIR, '3DOP_splits/' + train_config['train_dataset']) else: DATASET_SPLIT_FILE = args.dataset_split_file config_complete = load_config(args.config_path) if 'train' in config_complete: config = config_complete['train'] else: config = config_complete # input function ============================================================== dataset = KittiDataset(os.path.join(DATASET_DIR, 'image/training/image_2'), os.path.join(DATASET_DIR, 'velodyne/training/velodyne/'), os.path.join(DATASET_DIR, 'calib/training/calib/'), os.path.join(DATASET_DIR, 'labels/training/label_2'), DATASET_SPLIT_FILE, num_classes=config['num_classes']) NUM_CLASSES = dataset.num_classes if 'NUM_TEST_SAMPLE' not in train_config:
def run(config_file): config = load_config(config_file) logger, log_dir = get_train_logger(config_file, config) all_transforms = {} all_transforms['train'] = get_transforms(config.transforms.train) all_transforms['valid'] = get_transforms(config.transforms.test) dataloaders = { phase: make_dataset( data_folder=config.data.train_dir, df_path=config.data.train_df_path, fold_dir=config.data.fold_dir, phase=phase, fold=config.data.params.fold, img_shape=(config.data.height, config.data.width), transforms=all_transforms[phase], batch_size=config.train.batch_size, num_workers=config.num_workers, ) for phase in ['train', 'valid'] } """ model = getattr(smp, config.model.arch)( encoder_name=config.model.encoder, encoder_weights=config.model.pretrained, classes=config.data.num_classes, activation=None, ) """ model = make_model( model_name=config.model.model_name, encoder=config.model.encoder, decoder=config.model.decoder, class_num=config.data.num_classes ) params = [ {'params': model.parameters(), 'lr': config.optimizer.params.lr}, #{'params': model.encoder.parameters(), 'lr': config.optimizer.params.encoder_lr}, ] optimizer = get_optimizer(config.optimizer.name, params) criterion = smp.utils.losses.BCEDiceLoss() scheduler = ReduceLROnPlateau(optimizer, factor=0.5, patience=0) #if config.train.mixup: # callbacks.append(MixupCallback()) #if config.train.cutmix: # callbacks.append(CutMixCallback()) device = torch.device(config.device) model.to(device) if device == 'cuda': os.environ['CUDA_VISIBLE_DEVICES'] = config.devices runner = Runner() runner.train( model=model, criterion=criterion, optimizer=optimizer, loaders=dataloaders, logger=logger, metrics=dice_coeff, scheduler=scheduler, device=device, epoch_num=config.train.epoch, log_dir = log_dir )
description='Repeated evaluation of PointGNN.') parser.add_argument('eval_config_path', type=str, help='Path to train_config') parser.add_argument('--dataset_root_dir', type=str, default='../dataset/kitti/', help='Path to KITTI dataset. Default="../dataset/kitti/"') args = parser.parse_args() eval_config = load_train_config(args.eval_config_path) DATASET_DIR = args.dataset_root_dir config_path = os.path.join(eval_config['train_dir'], eval_config['config_path']) while not os.path.isfile(config_path): print('No config file found in %s, waiting' % config_path) time.sleep(eval_config['eval_every_second']) config = load_config(config_path) if 'eval' in config: config = config['eval'] dataset = KittiDataset(os.path.join(DATASET_DIR, 'velodyne/val/'), os.path.join(DATASET_DIR, 'labels/val/'), num_classes=config['num_classes']) NUM_CLASSES = dataset.num_classes print(dataset) if 'NUM_TEST_SAMPLE' not in eval_config: NUM_TEST_SAMPLE = dataset.num_files else: if eval_config['NUM_TEST_SAMPLE'] < 0: NUM_TEST_SAMPLE = dataset.num_files else: NUM_TEST_SAMPLE = eval_config['NUM_TEST_SAMPLE']