def create_dataloader(logger):
DATA_PATH = os.path.join('../', 'data')
# create dataloader
train_set = KittiRCNNDataset(root_dir=DATA_PATH, npoints=cfg.RPN.NUM_POINTS, split=cfg.TRAIN.SPLIT, mode='TRAIN',
logger=logger,
classes=cfg.CLASSES,
rcnn_training_roi_dir=args.rcnn_training_roi_dir,
rcnn_training_feature_dir=args.rcnn_training_feature_dir,
gt_database_dir=args.gt_database)
train_loader = DataLoader(train_set, batch_size=args.batch_size, pin_memory=True,
num_workers=args.workers, shuffle=True, collate_fn=train_set.collate_batch,
drop_last=True)
if args.train_with_eval:
test_set = KittiRCNNDataset(root_dir=DATA_PATH, npoints=cfg.RPN.NUM_POINTS, split=cfg.TRAIN.VAL_SPLIT, mode='EVAL',
logger=logger,
classes=cfg.CLASSES,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir)
test_loader = DataLoader(test_set, batch_size=1, shuffle=True, pin_memory=True,
num_workers=args.workers, collate_fn=test_set.collate_batch)
else:
test_loader = None
return train_loader, test_loader
def create_dataloader(logger):
DATA_PATH = os.path.join('../../', 'data')
# create dataloader
mean_std = ([103.939, 116.779, 123.68], [1.0, 1.0, 1.0])
val_input_transform = standard_transforms.Compose([
extended_transforms.FlipChannels(),
standard_transforms.ToTensor(),
standard_transforms.Lambda(lambda x: x.mul_(255)),
standard_transforms.Normalize(*mean_std)
])
train_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
image_size=cfg.PSP.IMAGE_SIZE,
transform=val_input_transform,
split=cfg.TRAIN.SPLIT,
mode='TRAIN',
logger=logger,
classes=cfg.CLASSES,
rcnn_training_roi_dir=args.rcnn_training_roi_dir,
rcnn_training_feature_dir=args.rcnn_training_feature_dir,
gt_database_dir=args.gt_database,
bgr_file=args.rpn_bgr,
mean_covariance_file=args.rpn_mean_covariance)
train_loader = DataLoader(train_set,
batch_size=args.batch_size,
pin_memory=True,
num_workers=args.workers,
shuffle=True,
collate_fn=train_set.collate_batch,
drop_last=True)
if args.train_with_eval:
test_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
image_size=cfg.PSP.IMAGE_SIZE,
transform=val_input_transform,
split=cfg.TRAIN.VAL_SPLIT,
mode='EVAL',
logger=logger,
classes=cfg.CLASSES,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir)
test_loader = DataLoader(test_set,
batch_size=1,
shuffle=True,
pin_memory=True,
num_workers=args.workers,
collate_fn=test_set.collate_batch)
else:
test_loader = None
return train_loader, test_loader
def create_dataloader(logger):
# go to semester project folder to access data
# data folder contains the data we downloaded from KITTI
DATA_PATH = os.path.join('../../', 'data')
# create dataloader
train_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.TRAIN.SPLIT,
mode='TRAIN',
logger=logger,
classes=cfg.CLASSES,
rcnn_training_roi_dir=args.rcnn_training_roi_dir,
rcnn_training_feature_dir=args.rcnn_training_feature_dir,
gt_database_dir=args.gt_database
) # this is the pickled database we made using generate_gt_database.py
# Now that we have a torch_data.Dataset (train_set) we make a dataloader
# see here for info : https://pytorch.org/tutorials/beginner/data_loading_tutorial.html
train_loader = DataLoader(train_set,
batch_size=args.batch_size,
pin_memory=True,
num_workers=args.workers,
shuffle=True,
collate_fn=train_set.collate_batch,
drop_last=True)
if args.train_with_eval:
test_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.TRAIN.VAL_SPLIT,
mode='EVAL',
logger=logger,
classes=cfg.CLASSES,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir)
test_loader = DataLoader(test_set,
batch_size=1,
shuffle=True,
pin_memory=True,
num_workers=args.workers,
collate_fn=test_set.collate_batch)
else:
test_loader = None
return train_loader, test_loader
def create_dataloader(logger):
mode = 'TEST' if args.test else 'EVAL'
#DATA_PATH = os.path.join('..', 'data')
# calls from app/frame_handler
DATA_PATH = '../../test_dataset/0_drive_0064_sync'
# create dataloader
test_set = KittiRCNNDataset(root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.TEST.SPLIT,
mode=mode,
random_select=True,
rcnn_eval_roi_dir=None,
rcnn_eval_feature_dir=None,
classes=cfg.CLASSES,
logger=logger)
test_loader = DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=8,
collate_fn=test_set.collate_batch)
return test_loader
def create_dataloader(logger):
mode = 'TEST' if args.test else 'EVAL'
#DATA_PATH = os.path.join('..', 'data')
DATA_PATH = cfg.DATA_PATH
# create dataloader
test_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.TEST.SPLIT,
mode=mode,
random_select=args.random_select,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir,
classes=cfg.CLASSES,
logger=logger)
test_loader = DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers,
collate_fn=test_set.collate_batch)
return test_loader
def create_dataloader(logger):
mode = 'TEST' if args.test else 'EVAL'
if (mode == 'TEST'):
DATA_PATH = '/media/jionie/my_disk/Kaggle/Lyft/input/3d-object-detection-for-autonomous-vehicles/test_root'
else:
DATA_PATH = '/media/jionie/my_disk/Kaggle/Lyft/input/3d-object-detection-for-autonomous-vehicles/train_root'
# create dataloader
test_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.TEST.SPLIT,
mode=mode,
random_select=args.random_select,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir,
classes=cfg.CLASSES,
logger=logger)
test_loader = DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers,
collate_fn=test_set.collate_batch)
return test_loader
def create_dataloader(logger):
# create dataloader
train_set = KittiRCNNDataset(root_dir=DATA_PATH, npoints=cfg.RPN.NUM_POINTS, split=cfg.TRAIN.SPLIT, mode='TRAIN',
logger=logger, classes=cfg.CLASSES, noise=args.noise_kind, weakly_num=args.weakly_num)
train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, pin_memory=True,
num_workers=0, collate_fn=train_set.collate_batch,
drop_last=True)
test_set = KittiRCNNDataset(root_dir=DATA_PATH, npoints=cfg.RPN.NUM_POINTS, split=cfg.TRAIN.VAL_SPLIT, mode='EVAL',
logger=logger, classes=cfg.CLASSES)
test_loader = DataLoader(test_set, batch_size=1, shuffle=False, pin_memory=True,
num_workers=0, collate_fn=test_set.collate_batch)
return train_loader, test_loader
def create_dataloader_da(logger):
mode = 'TEST' if args.test else 'EVAL'
DATA_PATH = os.path.join('../', 'data')
source_test_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.DA.SOURCE.VAL_SPLIT,
mode=mode,
logger=logger,
classes=cfg.CLASSES,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir,
is_source=True)
source_test_loader = DataLoader(source_test_set,
batch_size=1,
shuffle=True,
pin_memory=True,
num_workers=args.workers,
collate_fn=source_test_set.collate_batch)
target_test_set = nuscenes2kittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.DA.TARGET.VAL_SPLIT,
mode='EVAL',
logger=logger,
classes=cfg.CLASSES,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir,
is_source=False)
target_test_loader = DataLoader(target_test_set,
batch_size=1,
shuffle=True,
pin_memory=True,
num_workers=args.workers,
collate_fn=target_test_set.collate_batch)
return source_test_loader, target_test_loader
def create_dataloader(logger):
mode = 'TEST' if args.test else 'EVAL'
DATA_PATH = os.path.join('../../', 'data')
# create dataloader
mean_std = ([103.939, 116.779, 123.68], [1.0, 1.0, 1.0])
val_input_transform = standard_transforms.Compose([
extended_transforms.FlipChannels(),
standard_transforms.ToTensor(),
standard_transforms.Lambda(lambda x: x.mul_(255)),
standard_transforms.Normalize(*mean_std)
])
test_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
image_size=cfg.PSP.IMAGE_SIZE,
transform=val_input_transform,
split=cfg.TEST.SPLIT,
mode=mode,
random_select=args.random_select,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir,
classes=cfg.CLASSES,
logger=logger,
bgr_file=args.rpn_bgr,
mean_covariance_file=args.rpn_mean_covariance)
test_loader = DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers,
collate_fn=test_set.collate_batch)
return test_loader
def create_dataloader(config, logger):
mode = 'TEST' if config['test'] else 'EVAL'
DATA_PATH = os.path.join('..', 'data')
# create dataloader
test_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.TEST.SPLIT,
mode=mode,
random_select=config['random_select'],
rcnn_eval_roi_dir=config['rcnn_eval_roi_dir'],
rcnn_eval_feature_dir=config['rcnn_eval_feature_dir'],
classes=cfg.CLASSES,
logger=logger)
test_loader = DataLoader(test_set,
batch_size=config['batch_size'],
shuffle=False,
pin_memory=True,
num_workers=config['workers'],
collate_fn=test_set.collate_batch)
return test_loader
def create_dataloader(logger):
mode = 'TEST' if args.test else 'EVAL'
DATA_PATH = os.path.join('/raid/meng/Dataset/Kitti/object')
if args.eval_all:
print('Args eval_all enabled, small_val set will be used')
cfg.TEST.SPLIT = 'small_val'
# create dataloader
test_set = KittiRCNNDataset(root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.TEST.SPLIT,
mode=mode,
random_select=args.random_select,
classes=cfg.CLASSES,
logger=logger) #,noise='label_noise')
test_loader = DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers,
collate_fn=test_set.collate_batch)
return test_loader
logger = create_logger(log_file)
"""
dataset = KittiRCNNDataset(root_dir=DATA_PATH, npoints=cfg.RPN.NUM_POINTS, split=cfg.TEST.SPLIT, mode=mode,
random_select=False,
rcnn_eval_roi_dir=None,
rcnn_eval_feature_dir=None,
classes=cfg.CLASSES,
logger=logger)
dataset[0]
"""
from lib.datasets.kitti_rcnn_dataset import KittiRCNNDataset
train_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.TRAIN.SPLIT,
mode='TRAIN',
logger=logger,
classes=cfg.CLASSES,
rcnn_training_roi_dir=None,
rcnn_training_feature_dir=None,
gt_database_dir='tools/gt_database/train_gt_database_3level_Car.pkl')
batch = [train_set[0]]
batch_size = 1
ans_dict = {}
for key in batch[0].keys():
if cfg.RPN.ENABLED and key == 'gt_boxes3d' or \
(cfg.RCNN.ENABLED and cfg.RCNN.ROI_SAMPLE_JIT and key in ['gt_boxes3d', 'roi_boxes3d']):
max_gt = 0
for k in range(batch_size):
max_gt = max(max_gt, batch[k][key].__len__())
def create_dataloader(logger):
DATA_PATH = os.path.join('../', 'data')
# create dataloader
source_train_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.DA.SOURCE.SPLIT,
mode='TRAIN',
logger=logger,
classes=cfg.CLASSES,
rcnn_training_roi_dir=args.rcnn_training_roi_dir,
rcnn_training_feature_dir=args.rcnn_training_feature_dir,
gt_database_dir=args.gt_database,
is_source=True)
source_train_loader = DataLoader(source_train_set,
batch_size=args.batch_size,
pin_memory=True,
num_workers=args.workers,
shuffle=True,
collate_fn=source_train_set.collate_batch,
drop_last=True)
if cfg.DA.ENABLED:
target_train_set = nuscenes2kittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.DA.TARGET.SPLIT,
mode='TRAIN',
logger=logger,
classes=cfg.CLASSES,
rcnn_training_roi_dir=args.rcnn_training_roi_dir,
rcnn_training_feature_dir=args.rcnn_training_feature_dir,
gt_database_dir=args.gt_database,
is_source=False)
target_train_loader = DataLoader(
target_train_set,
batch_size=args.batch_size,
pin_memory=True,
num_workers=args.workers,
shuffle=True,
collate_fn=target_train_set.collate_batch,
drop_last=True)
if args.train_with_eval:
source_test_set = KittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.DA.SOURCE.VAL_SPLIT,
mode='EVAL',
logger=logger,
classes=cfg.CLASSES,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir,
is_source=True)
source_test_loader = DataLoader(
source_test_set,
batch_size=1,
shuffle=True,
pin_memory=True,
num_workers=args.workers,
collate_fn=source_test_set.collate_batch)
if cfg.DA.ENABLED:
target_test_set = nuscenes2kittiRCNNDataset(
root_dir=DATA_PATH,
npoints=cfg.RPN.NUM_POINTS,
split=cfg.DA.TARGET.VAL_SPLIT,
mode='EVAL',
logger=logger,
classes=cfg.CLASSES,
rcnn_eval_roi_dir=args.rcnn_eval_roi_dir,
rcnn_eval_feature_dir=args.rcnn_eval_feature_dir,
is_source=False)
target_test_loader = DataLoader(
target_test_set,
batch_size=1,
shuffle=True,
pin_memory=True,
num_workers=args.workers,
collate_fn=target_test_set.collate_batch)
else:
source_test_loader = None
target_test_loader = None
return source_train_loader, target_train_loader, source_test_loader, target_test_loader