def split_folds(config_file: str):
config = load_config(config_file)
df = pd.read_csv(config.data.train_df_path)
df = df[~df['EncodedPixels'].isnull()]
df['file_name'] = df['Image_Label'].map(lambda x: x.split('_')[0])
df['label'] = df['Image_Label'].map(lambda x: x.split('_')[-1])
classes = df['label'].unique()
df_g = df.groupby('file_name')['label'].agg(set).reset_index()
for class_name in classes:
df_g[class_name] = df_g['label'].map(
lambda x: 1 if class_name in x else 0)
# categorical
df_g['categorical'] = df_g.apply(
lambda x: str(x.Fish) + str(x.Flower) +
str(x.Sugar) + str(x.Gravel) , axis=1)
le = preprocessing.LabelEncoder()
df_g['categorical_num'] = le.fit_transform(df_g['categorical'])
df = pd.merge(df, df_g[['file_name', 'categorical_num']],
how='left', on='file_name')
kf = StratifiedKFold(n_splits=config.data.params.num_folds)
for i, (train_id, val_id) in enumerate(
kf.split(df.file_name, df.categorical_num)):
train = df.loc[train_id][['file_name']].drop_duplicates()
train.to_csv(f'{config.data.fold_dir}/'
f'{config.data.fold_train_file}_{i}.csv',
index=False)
val = df.loc[val_id][['file_name']].drop_duplicates()
val.to_csv(f'{config.data.fold_dir}/'
f'{config.data.fold_valid_file}_{i}.csv',
index=False)
IS_TEST = args.test
USE_BOX_MERGE = args.use_box_merge
USE_BOX_SCORE = args.use_box_score
DATASET_DIR = args.dataset_root_dir
if args.dataset_split_file == '':
DATASET_SPLIT_FILE = os.path.join(DATASET_DIR, './3DOP_splits/val.txt')
else:
DATASET_SPLIT_FILE = args.dataset_split_file
if args.output_dir == '':
OUTPUT_DIR = os.path.join(args.checkpoint_path, './eval/')
else:
OUTPUT_DIR = args.output_dir
CHECKPOINT_PATH = args.checkpoint_path
CONFIG_PATH = os.path.join(CHECKPOINT_PATH, 'config')
assert os.path.isfile(CONFIG_PATH), 'No config file found in %s'
config = load_config(CONFIG_PATH)
# setup dataset ===============================================================
if IS_TEST:
dataset = KittiDataset(os.path.join(DATASET_DIR, 'image/testing/image_2'),
os.path.join(DATASET_DIR,
'velodyne/testing/velodyne/'),
os.path.join(DATASET_DIR, 'calib/testing/calib/'),
'',
num_classes=config['num_classes'],
is_training=False)
else:
dataset = KittiDataset(os.path.join(DATASET_DIR, 'image/training/image_2'),
os.path.join(DATASET_DIR,
'velodyne/training/velodyne/'),
os.path.join(DATASET_DIR, 'calib/training/calib/'),
os.path.join(DATASET_DIR,
parser.add_argument('--dataset_split_file',
type=str,
default='',
help='Path to KITTI dataset split file.'
'Default="DATASET_ROOT_DIR/3DOP_splits'
'/train_config["train_dataset"]"')
args = parser.parse_args()
train_config = load_train_config(args.train_config_path)
DATASET_DIR = args.dataset_root_dir
if args.dataset_split_file == '':
DATASET_SPLIT_FILE = os.path.join(
DATASET_DIR, '3DOP_splits/' + train_config['train_dataset'])
else:
DATASET_SPLIT_FILE = args.dataset_split_file
config_complete = load_config(args.config_path)
if 'train' in config_complete:
config = config_complete['train']
else:
config = config_complete
# input function ==============================================================
dataset = KittiDataset(os.path.join(DATASET_DIR, 'image/training/image_2'),
os.path.join(DATASET_DIR,
'velodyne/training/velodyne/'),
os.path.join(DATASET_DIR, 'calib/training/calib/'),
os.path.join(DATASET_DIR, 'labels/training/label_2'),
DATASET_SPLIT_FILE,
num_classes=config['num_classes'])
NUM_CLASSES = dataset.num_classes
if 'NUM_TEST_SAMPLE' not in train_config:
def run(config_file):
config = load_config(config_file)
logger, log_dir = get_train_logger(config_file, config)
all_transforms = {}
all_transforms['train'] = get_transforms(config.transforms.train)
all_transforms['valid'] = get_transforms(config.transforms.test)
dataloaders = {
phase: make_dataset(
data_folder=config.data.train_dir,
df_path=config.data.train_df_path,
fold_dir=config.data.fold_dir,
phase=phase,
fold=config.data.params.fold,
img_shape=(config.data.height, config.data.width),
transforms=all_transforms[phase],
batch_size=config.train.batch_size,
num_workers=config.num_workers,
)
for phase in ['train', 'valid']
}
"""
model = getattr(smp, config.model.arch)(
encoder_name=config.model.encoder,
encoder_weights=config.model.pretrained,
classes=config.data.num_classes,
activation=None,
)
"""
model = make_model(
model_name=config.model.model_name,
encoder=config.model.encoder,
decoder=config.model.decoder,
class_num=config.data.num_classes
)
params = [
{'params': model.parameters(), 'lr': config.optimizer.params.lr},
#{'params': model.encoder.parameters(), 'lr': config.optimizer.params.encoder_lr},
]
optimizer = get_optimizer(config.optimizer.name, params)
criterion = smp.utils.losses.BCEDiceLoss()
scheduler = ReduceLROnPlateau(optimizer, factor=0.5, patience=0)
#if config.train.mixup:
# callbacks.append(MixupCallback())
#if config.train.cutmix:
# callbacks.append(CutMixCallback())
device = torch.device(config.device)
model.to(device)
if device == 'cuda':
os.environ['CUDA_VISIBLE_DEVICES'] = config.devices
runner = Runner()
runner.train(
model=model,
criterion=criterion,
optimizer=optimizer,
loaders=dataloaders,
logger=logger,
metrics=dice_coeff,
scheduler=scheduler,
device=device,
epoch_num=config.train.epoch,
log_dir = log_dir
)
description='Repeated evaluation of PointGNN.')
parser.add_argument('eval_config_path', type=str, help='Path to train_config')
parser.add_argument('--dataset_root_dir',
type=str,
default='../dataset/kitti/',
help='Path to KITTI dataset. Default="../dataset/kitti/"')
args = parser.parse_args()
eval_config = load_train_config(args.eval_config_path)
DATASET_DIR = args.dataset_root_dir
config_path = os.path.join(eval_config['train_dir'],
eval_config['config_path'])
while not os.path.isfile(config_path):
print('No config file found in %s, waiting' % config_path)
time.sleep(eval_config['eval_every_second'])
config = load_config(config_path)
if 'eval' in config:
config = config['eval']
dataset = KittiDataset(os.path.join(DATASET_DIR, 'velodyne/val/'),
os.path.join(DATASET_DIR, 'labels/val/'),
num_classes=config['num_classes'])
NUM_CLASSES = dataset.num_classes
print(dataset)
if 'NUM_TEST_SAMPLE' not in eval_config:
NUM_TEST_SAMPLE = dataset.num_files
else:
if eval_config['NUM_TEST_SAMPLE'] < 0:
NUM_TEST_SAMPLE = dataset.num_files
else:
NUM_TEST_SAMPLE = eval_config['NUM_TEST_SAMPLE']