def get_datasets(args, test_seed_offset=0): """ Gets training and test datasets. """ # Load superpoints graphs testlist, trainlist = [], [] #Changed by Arthur# # for n in range(1,7): for n in ["custom"]: if n != args.cvfold: #Changed by Arthur# # path = '{}/superpoint_graphs/Area_{:d}/'.format(args.S3DIS_PATH, n) path = '{}/superpoint_graphs/Area_{}/'.format(args.S3DIS_PATH, n) for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): trainlist.append(spg.spg_reader(args, path + fname, True)) #Changed by Arthur# # path = '{}/superpoint_graphs/Area_{:d}/'.format(args.S3DIS_PATH, args.cvfold) path = '{}/superpoint_graphs/Area_{}/'.format(args.S3DIS_PATH, "custom") for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): testlist.append(spg.spg_reader(args, path + fname, True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist = spg.scaler01(trainlist, testlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.S3DIS_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.S3DIS_PATH, test_seed_offset=test_seed_offset))
def get_datasets(args, test_seed_offset=0): """ Gets training and test datasets. """ # Load superpoints graphs testlist, trainlist = [], [] if args.data_mode == 'voxel': path = os.path.join(args.S3DIS_PATH, 'graph_v') else: path = os.path.join(args.S3DIS_PATH, 'graph_p') for fname in sorted(os.listdir(path)): area = int(fname[5]) if fname.endswith(".h5"): if area != args.cvfold: trainlist.append( spg.spg_reader(args, os.path.join(path, fname), True)) elif area == args.cvfold: testlist.append( spg.spg_reader(args, os.path.join(path, fname), True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist = spg.scaler01(trainlist, testlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.S3DIS_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.S3DIS_PATH, test_seed_offset=test_seed_offset))
def get_datasets(args, test_seed_offset=0): """build training and testing set For the ONERD, only the test dataset is available """ #for a simple test organization testset = [ 'formatted/' + f for f in os.listdir(args.CUSTOM_SET_PATH + '/superpoint_graphs/formatted') ] # Load superpoints graphs testlist, trainlist = [], [] for n in testset: testlist.append( spg.spg_reader(args, args.CUSTOM_SET_PATH + '/superpoint_graphs/' + n, True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist = spg.scaler01(trainlist, testlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.CUSTOM_SET_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.CUSTOM_SET_PATH, test_seed_offset=test_seed_offset))
def get_datasets(args, test_seed_offset=0): train_names = ['bildstein_station1', 'bildstein_station5', 'domfountain_station1', 'domfountain_station3', 'neugasse_station1', 'sg27_station1', 'sg27_station2', 'sg27_station5', 'sg27_station9', 'sg28_station4', 'untermaederbrunnen_station1'] valid_names = ['bildstein_station3', 'domfountain_station2', 'sg27_station4', 'untermaederbrunnen_station3'] if args.db_train_name == 'train': trainset = ['train/' + f for f in train_names] elif args.db_train_name == 'trainval': trainset = ['train/' + f for f in train_names + valid_names] if args.db_test_name == 'val': testset = ['train/' + f for f in valid_names] elif args.db_test_name == 'testred': testset = ['test_reduced/' + os.path.splitext(f)[0] for f in os.listdir(args.SEMA3D_PATH + '/superpoint_graphs/test_reduced')] elif args.db_test_name == 'testfull': testset = ['test_full/' + os.path.splitext(f)[0] for f in os.listdir(args.SEMA3D_PATH + '/superpoint_graphs/test_full')] # Load superpoints graphs testlist, trainlist = [], [] for n in trainset: trainlist.append(spg.spg_reader(args, args.SEMA3D_PATH + '/superpoint_graphs/' + n + '.h5', True)) for n in testset: testlist.append(spg.spg_reader(args, args.SEMA3D_PATH + '/superpoint_graphs/' + n + '.h5', True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist = spg.scaler01(trainlist, testlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.SEMA3D_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.SEMA3D_PATH, test_seed_offset=test_seed_offset))
def get_datasets(args, test_seed_offset=0): """ Gets training and test datasets. """ # Load superpoints graphs testlist, trainlist, validlist = [], [], [] valid_names = ['0001_00000.h5','0001_00085.h5', '0001_00170.h5','0001_00230.h5','0001_00325.h5','0001_00420.h5', \ '0002_00000.h5','0002_00111.h5','0002_00223.h5','0018_00030.h5','0018_00184.h5','0018_00338.h5',\ '0020_00080.h5','0020_00262.h5','0020_00444.h5','0020_00542.h5','0020_00692.h5', '0020_00800.h5'] for n in range(1,7): if n != args.cvfold: path = '{}/superpoint_graphs/0{:d}/'.format(args.VKITTI_PATH, n) for fname in sorted(os.listdir(path)): if fname.endswith(".h5") and not (args.use_val_set and fname in valid_names): #training set trainlist.append(spg.spg_reader(args, path + fname, True)) if fname.endswith(".h5") and (args.use_val_set and fname in valid_names): #validation set validlist.append(spg.spg_reader(args, path + fname, True)) path = '{}/superpoint_graphs/0{:d}/'.format(args.VKITTI_PATH, args.cvfold) #evaluation set for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): testlist.append(spg.spg_reader(args, path + fname, True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist, validlist = spg.scaler01(trainlist, testlist, validlist=validlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.VKITTI_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.VKITTI_PATH, test_seed_offset=test_seed_offset)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in validlist], functools.partial(spg.loader, train=False, args=args, db_path=args.VKITTI_PATH, test_seed_offset=test_seed_offset))
def get_datasets(args, test_seed_offset=0): """ Gets training and test datasets. """ # Load superpoints graphs testlist, trainlist, validlist = [], [], [] # valid_names = ['000000.h5','000100.h5', '000200.h5','000300.h5','000400.h5','000500.h5', \ # '000600.h5','000700.h5','000800.h5','000900.h5','001000.h5','001100.h5','001200.h5','001300.h5','001400.h5', \ # '001500.h5','001600.h5','001700.h5','001800.h5','001900.h5','002000.h5','002100.h5','002200.h5','002300.h5'] # data_set_list = [0,1,2,3,4,5,6,7,8,9,10,90,91,92,93] # for n in data_set_list: # if n != args.cvfold: # path = '{}/superpoint_graphs/{:0>2d}/'.format(args.SKITTI_PATH, n) # for fname in sorted(os.listdir(path)): # if fname.endswith(".h5") and not (args.use_val_set and fname in valid_names): # #training set # trainlist.append(spg.spg_reader(args, path + fname, True)) # if fname.endswith(".h5") and (args.use_val_set and fname in valid_names): # #validation set # validlist.append(spg.spg_reader(args, path + fname, True)) # train train_set_list = [90] for n in train_set_list: path = '{}/superpoint_graphs/{:0>2d}/'.format(args.SKITTI_PATH, n) #train set for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): trainlist.append(spg.spg_reader(args, path + fname, True)) # val val_set_list = [8] for n in val_set_list: path = '{}/superpoint_graphs/{:0>2d}/'.format(args.SKITTI_PATH, n) #val set for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): validlist.append(spg.spg_reader(args, path + fname, True)) # test test_set_list = [10] for n in test_set_list: path = '{}/superpoint_graphs/{:0>2d}/'.format(args.SKITTI_PATH, n) #test set for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): testlist.append(spg.spg_reader(args, path + fname, True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist, validlist, scaler = spg.scaler01(trainlist, testlist, validlist=validlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.SKITTI_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.SKITTI_PATH, test_seed_offset=test_seed_offset)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in validlist], functools.partial(spg.loader, train=False, args=args, db_path=args.SKITTI_PATH, test_seed_offset=test_seed_offset)), \ scaler
def get_datasets(args, test_seed_offset=0): """ Gets training and test datasets. """ # Load superpoints graphs testlist, trainlist, validlist = [], [], [] valid_names = ['hallway_1.h5', 'hallway_6.h5', 'hallway_11.h5', 'office_1.h5' \ , 'office_6.h5', 'office_11.h5', 'office_16.h5', 'office_21.h5', 'office_26.h5' \ , 'office_31.h5', 'office_36.h5'\ ,'WC_2.h5', 'storage_1.h5', 'storage_5.h5', 'conferenceRoom_2.h5', 'auditorium_1.h5'] #if args.db_test_name == 'test' then the test set is the evaluation set #otherwise it serves as valdiation set to select the best epoch for n in range(1, 7): if n != args.cvfold: path = '{}/superpoint_graphs/Area_{:d}/'.format(args.S3DIS_PATH, n) for fname in sorted(os.listdir(path)): if fname.endswith(".h5") and not (args.use_val_set and fname in valid_names): #training set trainlist.append(spg.spg_reader(args, path + fname, True)) if fname.endswith(".h5") and (args.use_val_set and fname in valid_names): #validation set validlist.append(spg.spg_reader(args, path + fname, True)) path = '{}/superpoint_graphs/Area_{:d}/'.format(args.S3DIS_PATH, args.cvfold) #evaluation set for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): testlist.append(spg.spg_reader(args, path + fname, True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist, validlist = spg.scaler01(trainlist, testlist, validlist=validlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.S3DIS_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.S3DIS_PATH, test_seed_offset=test_seed_offset)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in validlist], functools.partial(spg.loader, train=False, args=args, db_path=args.S3DIS_PATH, test_seed_offset=test_seed_offset))
def get_datasets(args, test_seed_offset=0): """build training and testing set""" #for a simple train/test organization trainset = ['train/' + f for f in os.listdir(args.TANKER_PATH + '/superpoint_graphs'+num+'/train')] testset = ['test/' + f for f in os.listdir(args.TANKER_PATH + '/superpoint_graphs'+num+'/test')] # Load superpoints graphs testlist, trainlist = [], [] for n in trainset: trainlist.append(spg.spg_reader(args, args.TANKER_PATH + '/superpoint_graphs'+num+'/' + n, True)) for n in testset: testlist.append(spg.spg_reader(args, args.TANKER_PATH + '/superpoint_graphs'+num+'/' + n , True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist = spg.scaler01(trainlist, testlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.TANKER_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.TANKER_PATH, test_seed_offset=test_seed_offset))
def get_datasets(args, test_seed_offset=0): """build training and testing set""" #for a simple train/test organization # trainset = ['trainval/' + f for f in os.listdir(args.AERIAL7_PATH + '/superpoint_graphs/train')] # testset = ['test/' + f for f in os.listdir(args.AERIAL7_PATH + '/superpoint_graphs/test')] # #Load superpoints graphs # testlist, trainlist = [], [] # for n in trainset: # trainlist.append(spg.spg_reader(args, args.AERIAL7_PATH + '/superpoint_graphs/' + n, True)) # for n in testset: # testlist.append(spg.spg_reader(args, args.AERIAL7_PATH + '/superpoint_graphs/' + n, True)) testlist, trainlist = [], [] for n in range(1, 7): if n != args.cvfold: path = '{}/superpoint_graphs/Area_{:d}/'.format( args.AERIAL7_PATH, n) for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): trainlist.append(spg.spg_reader(args, path + fname, True)) path = '{}/superpoint_graphs/Area_{:d}/'.format(args.AERIAL7_PATH, args.cvfold) for fname in sorted(os.listdir(path)): if fname.endswith(".h5"): testlist.append(spg.spg_reader(args, path + fname, True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist = spg.scaler01(trainlist, testlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.AERIAL7_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.AERIAL7_PATH, test_seed_offset=test_seed_offset))
def get_datasets(args, test_seed_offset=0): train_names = [ '6755_66525.h5', '6955_66645.h5', '6875_66585.h5', '6820_66550.h5', '6950_66640.h5', '6955_66650.h5', '7055_66455.h5', '7020_66300.h5', '7050_66335.h5', '6825_66560.h5', '7015_66550.h5', '7055_66350.h5', '7030_66525.h5', '6985_66650.h5', '7005_66575.h5', '6750_66520.h5', '6770_66530.h5', '6990_66595.h5', '6760_66530.h5', '6805_66540.h5', '7000_66580.h5', '6980_66650.h5', '7060_66385.h5', '7055_66410.h5', '7010_66565.h5', '7015_66555.h5', '6810_66545.h5', '7040_66495.h5', '6870_66580.h5', '7005_66565.h5', '7045_66485.h5', '7055_66445.h5', '7025_66530.h5', '6835_66565.h5', '7050_66340.h5', '7030_66315.h5', '7035_66325.h5', '6985_66605.h5', '7020_66305.h5', '7040_66505.h5', '6875_66580.h5', '6965_66630.h5', '7015_66560.h5', '7045_66500.h5', '7055_66340.h5', '7060_66420.h5', '6995_66575.h5', '7060_66400.h5', '6800_66540.h5', '7015_66545.h5', '7040_66325.h5', '7020_66310.h5', '7050_66330.h5', '7055_66420.h5', '6900_66605.h5', '6890_66600.h5', '6830_66560.h5', '6790_66540.h5', '6840_66565.h5', '6915_66610.h5', '7020_66540.h5', '7000_66570.h5', '6855_66570.h5', '7045_66335.h5', '6785_66535.h5', '6870_66585.h5', '6950_66645.h5', '6845_66570.h5', '6980_66610.h5', '6990_66650.h5', '6905_66610.h5', '6860_66575.h5', '7060_66360.h5', '6940_66635.h5', '6765_66530.h5', '6935_66635.h5', '6780_66530.h5', '6975_66650.h5', '7055_66345.h5', '7035_66520.h5', '6970_66620.h5', '7005_66570.h5', '6905_66605.h5', '6935_66625.h5', '7020_66550.h5', '7060_66395.h5', '6930_66625.h5', '7050_66450.h5', '7025_66305.h5', '7050_66475.h5', '6900_66600.h5', '6850_66565.h5', '6865_66575.h5', '7045_66495.h5', '6990_66590.h5', '6795_66540.h5', '7035_66320.h5', '7055_66460.h5', '6975_66615.h5', '6910_66610.h5', '7060_66380.h5', '7055_66335.h5', '6845_66565.h5', '6965_66635.h5', '6775_66530.h5', '6750_66525.h5', '6815_66545.h5', '6975_66620.h5' ] #'6875_66590.h5',, '6960_66635.h5' valid_names = [ '6835_66560.h5', '6960_66640.h5', '6880_66595.h5', '6860_66580.h5', '6955_66640.h5', '6795_66535.h5' ] if args.db_train_name == 'train': trainset = ['train/' + f for f in train_names] elif args.db_train_name == 'trainval': trainset = ['train/' + f for f in train_names + valid_names] validset = [] testset = [] if args.use_val_set: validset = ['train/' + f for f in valid_names] if args.db_test_name == 'testred': testset = [ 'test_reduced/' + os.path.splitext(f)[0] for f in os.listdir(args.SEMA3D_PATH + '/superpoint_graphs/test_reduced') ] elif args.db_test_name == 'testfull': testset = [ 'test_full/' + os.path.splitext(f)[0] for f in os.listdir(args.SEMA3D_PATH + '/superpoint_graphs/test_full') ] # Load superpoints graphs testlist, trainlist, validlist = [], [], [] for n in trainset: trainlist.append( spg.spg_reader(args, args.SEMA3D_PATH + '/superpoint_graphs/' + n, True)) for n in validset: validlist.append( spg.spg_reader(args, args.SEMA3D_PATH + '/superpoint_graphs/' + n, True)) for n in testset: testlist.append( spg.spg_reader( args, args.SEMA3D_PATH + '/superpoint_graphs/' + n + '.h5', True)) # Normalize edge features if args.spg_attribs01: trainlist, testlist, validlist, scaler = spg.scaler01( trainlist, testlist, validlist=validlist) return tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in trainlist], functools.partial(spg.loader, train=True, args=args, db_path=args.SEMA3D_PATH)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in testlist], functools.partial(spg.loader, train=False, args=args, db_path=args.SEMA3D_PATH, test_seed_offset=test_seed_offset)), \ tnt.dataset.ListDataset([spg.spg_to_igraph(*tlist) for tlist in validlist], functools.partial(spg.loader, train=False, args=args, db_path=args.SEMA3D_PATH, test_seed_offset=test_seed_offset)),\ scaler