def load_snap_clouds(path, dataset, only_last=False):
cloud_folders = np.array([
join(path, f) for f in listdir_str(path) if f.startswith('val_preds')
])
cloud_epochs = np.array([int(f.split('_')[-1]) for f in cloud_folders])
epoch_order = np.argsort(cloud_epochs)
cloud_epochs = cloud_epochs[epoch_order]
cloud_folders = cloud_folders[epoch_order]
Confs = np.zeros(
(len(cloud_epochs), dataset.num_classes, dataset.num_classes),
dtype=np.int32)
for c_i, cloud_folder in enumerate(cloud_folders):
if only_last and c_i < len(cloud_epochs) - 1:
continue
# Load confusion if previously saved
conf_file = join(cloud_folder, 'conf.txt')
if isfile(conf_file):
Confs[c_i] += np.loadtxt(conf_file, dtype=np.int32)
else:
for f in listdir_str(cloud_folder):
if f.endswith('.ply') and not f.endswith('sub.ply'):
data = read_ply(join(cloud_folder, f))
labels = data['class']
preds = data['preds']
Confs[c_i] += fast_confusion(
labels, preds, dataset.label_values).astype(np.int32)
np.savetxt(conf_file, Confs[c_i], '%12d')
# Erase ply to save disk memory
if c_i < len(cloud_folders) - 1:
for f in listdir_str(cloud_folder):
if f.endswith('.ply'):
remove(join(cloud_folder, f))
# Remove ignored labels from confusions
for l_ind, label_value in reversed(list(enumerate(dataset.label_values))):
if label_value in dataset.ignored_labels:
Confs = np.delete(Confs, l_ind, axis=1)
Confs = np.delete(Confs, l_ind, axis=2)
return cloud_epochs, IoU_from_confusions(Confs)
def cloud_segmentation_test(self,
net,
test_loader,
config,
num_votes=100,
debug=False):
"""
Test method for cloud segmentation models
"""
############
# Initialize
############
# Choose test smoothing parameter (0 for no smothing, 0.99 for big smoothing)
test_smooth = 0.95
test_radius_ratio = 0.7
softmax = torch.nn.Softmax(1)
# Number of classes including ignored labels
nc_tot = test_loader.dataset.num_classes
# Number of classes predicted by the model
nc_model = config.num_classes
# Initiate global prediction over test clouds
self.test_probs = [
np.zeros((l.shape[0], nc_model))
for l in test_loader.dataset.input_labels
]
# Test saving path
if config.saving:
test_path = join('test', config.saving_path.split('/')[-1])
if not exists(test_path):
makedirs(test_path)
if not exists(join(test_path, 'predictions')):
makedirs(join(test_path, 'predictions'))
if not exists(join(test_path, 'probs')):
makedirs(join(test_path, 'probs'))
if not exists(join(test_path, 'potentials')):
makedirs(join(test_path, 'potentials'))
else:
test_path = None
# If on validation directly compute score
if test_loader.dataset.set == 'validation':
val_proportions = np.zeros(nc_model, dtype=np.float32)
i = 0
for label_value in test_loader.dataset.label_values:
if label_value not in test_loader.dataset.ignored_labels:
val_proportions[i] = np.sum([
np.sum(labels == label_value)
for labels in test_loader.dataset.validation_labels
])
i += 1
else:
val_proportions = None
#####################
# Network predictions
#####################
test_epoch = 0
last_min = -0.5
t = [time.time()]
last_display = time.time()
mean_dt = np.zeros(1)
# Start test loop
while True:
print('Initialize workers')
for i, batch in enumerate(test_loader):
# New time
t = t[-1:]
t += [time.time()]
if i == 0:
print('Done in {:.1f}s'.format(t[1] - t[0]))
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
t += [time.time()]
# Get probs and labels
stacked_probs = softmax(outputs).cpu().detach().numpy()
s_points = batch.points[0].cpu().numpy()
lengths = batch.lengths[0].cpu().numpy()
in_inds = batch.input_inds.cpu().numpy()
cloud_inds = batch.cloud_inds.cpu().numpy()
torch.cuda.synchronize(self.device)
# Get predictions and labels per instance
# ***************************************
i0 = 0
for b_i, length in enumerate(lengths):
# Get prediction
points = s_points[i0:i0 + length]
probs = stacked_probs[i0:i0 + length]
inds = in_inds[i0:i0 + length]
c_i = cloud_inds[b_i]
if 0 < test_radius_ratio < 1:
mask = np.sum(
points**2,
axis=1) < (test_radius_ratio * config.in_radius)**2
inds = inds[mask]
probs = probs[mask]
# Update current probs in whole cloud
self.test_probs[c_i][inds] = test_smooth * self.test_probs[
c_i][inds] + (1 - test_smooth) * probs
i0 += length
# Average timing
t += [time.time()]
if i < 2:
mean_dt = np.array(t[1:]) - np.array(t[:-1])
else:
mean_dt = 0.9 * mean_dt + 0.1 * (np.array(t[1:]) -
np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'e{:03d}-i{:04d} => {:.1f}% (timings : {:4.2f} {:4.2f} {:4.2f})'
print(
message.format(test_epoch, i,
100 * i / config.validation_size,
1000 * (mean_dt[0]),
1000 * (mean_dt[1]),
1000 * (mean_dt[2])))
# Update minimum od potentials
new_min = torch.min(test_loader.dataset.min_potentials)
print('Test epoch {:d}, end. Min potential = {:.1f}'.format(
test_epoch, new_min))
#print([np.mean(pots) for pots in test_loader.dataset.potentials])
# Save predicted cloud
if last_min + 1 < new_min:
# Update last_min
last_min += 1
# Show vote results (On subcloud so it is not the good values here)
if test_loader.dataset.set == 'validation':
print('\nConfusion on sub clouds')
Confs = []
for i, file_path in enumerate(test_loader.dataset.files):
# Insert false columns for ignored labels
probs = np.array(self.test_probs[i], copy=True)
for l_ind, label_value in enumerate(
test_loader.dataset.label_values):
if label_value in test_loader.dataset.ignored_labels:
probs = np.insert(probs, l_ind, 0, axis=1)
# Predicted labels
preds = test_loader.dataset.label_values[np.argmax(
probs, axis=1)].astype(np.int32)
# Targets
targets = test_loader.dataset.input_labels[i]
# Confs
Confs += [
fast_confusion(targets, preds,
test_loader.dataset.label_values)
]
# Regroup confusions
C = np.sum(np.stack(Confs), axis=0).astype(np.float32)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(
list(enumerate(test_loader.dataset.label_values))):
if label_value in test_loader.dataset.ignored_labels:
C = np.delete(C, l_ind, axis=0)
C = np.delete(C, l_ind, axis=1)
# Rescale with the right number of point per class
C *= np.expand_dims(
val_proportions / (np.sum(C, axis=1) + 1e-6), 1)
# Compute IoUs
IoUs = IoU_from_confusions(C)
mIoU = np.mean(IoUs)
s = '{:5.2f} | '.format(100 * mIoU)
for IoU in IoUs:
s += '{:5.2f} '.format(100 * IoU)
print(s + '\n')
# Save real IoU once in a while
if int(np.ceil(new_min)) % 10 == 0:
# Project predictions
print('\nReproject Vote #{:d}'.format(
int(np.floor(new_min))))
t1 = time.time()
proj_probs = []
for i, file_path in enumerate(test_loader.dataset.files):
print(i, file_path,
test_loader.dataset.test_proj[i].shape,
self.test_probs[i].shape)
print(test_loader.dataset.test_proj[i].dtype,
np.max(test_loader.dataset.test_proj[i]))
print(test_loader.dataset.test_proj[i][:5])
# Reproject probs on the evaluations points
probs = self.test_probs[i][
test_loader.dataset.test_proj[i], :]
proj_probs += [probs]
t2 = time.time()
print('Done in {:.1f} s\n'.format(t2 - t1))
# Show vote results
if test_loader.dataset.set == 'validation':
print('Confusion on full clouds')
t1 = time.time()
Confs = []
for i, file_path in enumerate(
test_loader.dataset.files):
# Insert false columns for ignored labels
for l_ind, label_value in enumerate(
test_loader.dataset.label_values):
if label_value in test_loader.dataset.ignored_labels:
proj_probs[i] = np.insert(proj_probs[i],
l_ind,
0,
axis=1)
# Get the predicted labels
preds = test_loader.dataset.label_values[np.argmax(
proj_probs[i], axis=1)].astype(np.int32)
# Confusion
targets = test_loader.dataset.validation_labels[i]
Confs += [
fast_confusion(
targets, preds,
test_loader.dataset.label_values)
]
t2 = time.time()
print('Done in {:.1f} s\n'.format(t2 - t1))
# Regroup confusions
C = np.sum(np.stack(Confs), axis=0)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(
list(
enumerate(
test_loader.dataset.label_values))):
if label_value in test_loader.dataset.ignored_labels:
C = np.delete(C, l_ind, axis=0)
C = np.delete(C, l_ind, axis=1)
IoUs = IoU_from_confusions(C)
mIoU = np.mean(IoUs)
s = '{:5.2f} | '.format(100 * mIoU)
for IoU in IoUs:
s += '{:5.2f} '.format(100 * IoU)
print('-' * len(s))
print(s)
print('-' * len(s) + '\n')
# Save predictions
print('Saving clouds')
t1 = time.time()
for i, file_path in enumerate(test_loader.dataset.files):
# Get file
points = test_loader.dataset.load_evaluation_points(
file_path)
# Get the predicted labels
preds = test_loader.dataset.label_values[np.argmax(
proj_probs[i], axis=1)].astype(np.int32)
# Save plys
cloud_name = file_path.split('/')[-1]
test_name = join(test_path, 'predictions', cloud_name)
write_ply(test_name, [points, preds],
['x', 'y', 'z', 'preds'])
test_name2 = join(test_path, 'probs', cloud_name)
prob_names = [
'_'.join(test_loader.dataset.label_to_names[label].
split())
for label in test_loader.dataset.label_values
]
write_ply(test_name2, [points, proj_probs[i]],
['x', 'y', 'z'] + prob_names)
# Save potentials
pot_points = np.array(
test_loader.dataset.pot_trees[i].data, copy=False)
pot_name = join(test_path, 'potentials', cloud_name)
pots = test_loader.dataset.potentials[i].numpy(
).astype(np.float32)
write_ply(pot_name,
[pot_points.astype(np.float32), pots],
['x', 'y', 'z', 'pots'])
# Save ascii preds
if test_loader.dataset.set == 'test':
if test_loader.dataset.name.startswith(
'Semantic3D'):
ascii_name = join(
test_path, 'predictions', test_loader.
dataset.ascii_files[cloud_name])
else:
ascii_name = join(test_path, 'predictions',
cloud_name[:-4] + '.txt')
np.savetxt(ascii_name, preds, fmt='%d')
t2 = time.time()
print('Done in {:.1f} s\n'.format(t2 - t1))
test_epoch += 1
# Break when reaching number of desired votes
if last_min > num_votes:
break
return
def slam_segmentation_test(self,
net,
test_loader,
config,
num_votes=100,
debug=True):
"""
Test method for slam segmentation models
"""
############
# Initialize
############
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
test_smooth = 0.5
last_min = -0.5
softmax = torch.nn.Softmax(1)
# Number of classes including ignored labels
nc_tot = test_loader.dataset.num_classes
nc_model = net.C
# Test saving path
test_path = None
report_path = None
if config.saving:
test_path = join('test', config.saving_path.split('/')[-1])
if not exists(test_path):
makedirs(test_path)
report_path = join(test_path, 'reports')
if not exists(report_path):
makedirs(report_path)
if test_loader.dataset.set == 'validation':
for folder in ['val_predictions', 'val_probs']:
if not exists(join(test_path, folder)):
makedirs(join(test_path, folder))
else:
for folder in ['predictions', 'probs']:
if not exists(join(test_path, folder)):
makedirs(join(test_path, folder))
# Init validation container
all_f_preds = []
all_f_labels = []
if test_loader.dataset.set == 'validation':
for i, seq_frames in enumerate(test_loader.dataset.frames):
all_f_preds.append(
[np.zeros((0, ), dtype=np.int32) for _ in seq_frames])
all_f_labels.append(
[np.zeros((0, ), dtype=np.int32) for _ in seq_frames])
#####################
# Network predictions
#####################
predictions = []
targets = []
test_epoch = 0
t = [time.time()]
last_display = time.time()
mean_dt = np.zeros(1)
# Start test loop
while True:
print('Initialize workers')
for i, batch in enumerate(test_loader):
# New time
t = t[-1:]
t += [time.time()]
if i == 0:
print('Done in {:.1f}s'.format(t[1] - t[0]))
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
# Get probs and labels
stk_probs = softmax(outputs).cpu().detach().numpy()
lengths = batch.lengths[0].cpu().numpy()
f_inds = batch.frame_inds.cpu().numpy()
r_inds_list = batch.reproj_inds
r_mask_list = batch.reproj_masks
labels_list = batch.val_labels
torch.cuda.synchronize(self.device)
t += [time.time()]
# Get predictions and labels per instance
# ***************************************
i0 = 0
for b_i, length in enumerate(lengths):
# Get prediction
probs = stk_probs[i0:i0 + length]
proj_inds = r_inds_list[b_i]
proj_mask = r_mask_list[b_i]
frame_labels = labels_list[b_i]
s_ind = f_inds[b_i, 0]
f_ind = f_inds[b_i, 1]
# Project predictions on the frame points
proj_probs = probs[proj_inds]
# Safe check if only one point:
if proj_probs.ndim < 2:
proj_probs = np.expand_dims(proj_probs, 0)
# Save probs in a binary file (uint8 format for lighter weight)
seq_name = test_loader.dataset.sequences[s_ind]
if test_loader.dataset.set == 'validation':
folder = 'val_probs'
pred_folder = 'val_predictions'
else:
folder = 'probs'
pred_folder = 'predictions'
filename = '{:s}_{:07d}.npy'.format(seq_name, f_ind)
filepath = join(test_path, folder, filename)
if exists(filepath):
frame_probs_uint8 = np.load(filepath)
else:
frame_probs_uint8 = np.zeros(
(proj_mask.shape[0], nc_model), dtype=np.uint8)
frame_probs = frame_probs_uint8[proj_mask, :].astype(
np.float32) / 255
frame_probs = test_smooth * frame_probs + (
1 - test_smooth) * proj_probs
frame_probs_uint8[proj_mask, :] = (frame_probs *
255).astype(np.uint8)
np.save(filepath, frame_probs_uint8)
# Save some prediction in ply format for visual
if test_loader.dataset.set == 'validation':
# Insert false columns for ignored labels
frame_probs_uint8_bis = frame_probs_uint8.copy()
for l_ind, label_value in enumerate(
test_loader.dataset.label_values):
if label_value in test_loader.dataset.ignored_labels:
frame_probs_uint8_bis = np.insert(
frame_probs_uint8_bis, l_ind, 0, axis=1)
# Predicted labels
frame_preds = test_loader.dataset.label_values[
np.argmax(frame_probs_uint8_bis,
axis=1)].astype(np.int32)
# Save some of the frame pots
if f_ind % 20 == 0:
seq_path = join(
test_loader.dataset.path, 'sequences',
test_loader.dataset.sequences[s_ind])
velo_file = join(
seq_path, 'velodyne',
test_loader.dataset.frames[s_ind][f_ind] +
'.bin')
frame_points = np.fromfile(velo_file,
dtype=np.float32)
frame_points = frame_points.reshape((-1, 4))
predpath = join(test_path, pred_folder,
filename[:-4] + '.ply')
#pots = test_loader.dataset.f_potentials[s_ind][f_ind]
pots = np.zeros((0, ))
if pots.shape[0] > 0:
write_ply(predpath, [
frame_points[:, :3], frame_labels,
frame_preds, pots
], ['x', 'y', 'z', 'gt', 'pre', 'pots'])
else:
write_ply(predpath, [
frame_points[:, :3], frame_labels,
frame_preds
], ['x', 'y', 'z', 'gt', 'pre'])
# Also Save lbl probabilities
probpath = join(test_path, folder,
filename[:-4] + '_probs.ply')
lbl_names = [
test_loader.dataset.label_to_names[l]
for l in test_loader.dataset.label_values
if l not in test_loader.dataset.ignored_labels
]
write_ply(probpath,
[frame_points[:, :3], frame_probs_uint8],
['x', 'y', 'z'] + lbl_names)
# keep frame preds in memory
all_f_preds[s_ind][f_ind] = frame_preds
all_f_labels[s_ind][f_ind] = frame_labels
else:
# Save some of the frame preds
if f_inds[b_i, 1] % 100 == 0:
# Insert false columns for ignored labels
for l_ind, label_value in enumerate(
test_loader.dataset.label_values):
if label_value in test_loader.dataset.ignored_labels:
frame_probs_uint8 = np.insert(
frame_probs_uint8, l_ind, 0, axis=1)
# Predicted labels
frame_preds = test_loader.dataset.label_values[
np.argmax(frame_probs_uint8,
axis=1)].astype(np.int32)
# Load points
seq_path = join(
test_loader.dataset.path, 'sequences',
test_loader.dataset.sequences[s_ind])
velo_file = join(
seq_path, 'velodyne',
test_loader.dataset.frames[s_ind][f_ind] +
'.bin')
frame_points = np.fromfile(velo_file,
dtype=np.float32)
frame_points = frame_points.reshape((-1, 4))
predpath = join(test_path, pred_folder,
filename[:-4] + '.ply')
#pots = test_loader.dataset.f_potentials[s_ind][f_ind]
pots = np.zeros((0, ))
if pots.shape[0] > 0:
write_ply(
predpath,
[frame_points[:, :3], frame_preds, pots],
['x', 'y', 'z', 'pre', 'pots'])
else:
write_ply(predpath,
[frame_points[:, :3], frame_preds],
['x', 'y', 'z', 'pre'])
# Stack all prediction for this epoch
i0 += length
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) -
np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'e{:03d}-i{:04d} => {:.1f}% (timings : {:4.2f} {:4.2f} {:4.2f}) / pots {:d} => {:.1f}%'
min_pot = int(
torch.floor(torch.min(test_loader.dataset.potentials)))
pot_num = torch.sum(
test_loader.dataset.potentials > min_pot + 0.5).type(
torch.int32).item()
current_num = pot_num + (
i + 1 - config.validation_size) * config.val_batch_num
print(
message.format(
test_epoch, i, 100 * i / config.validation_size,
1000 * (mean_dt[0]), 1000 * (mean_dt[1]),
1000 * (mean_dt[2]), min_pot, 100.0 * current_num /
len(test_loader.dataset.potentials)))
# Update minimum od potentials
new_min = torch.min(test_loader.dataset.potentials)
print('Test epoch {:d}, end. Min potential = {:.1f}'.format(
test_epoch, new_min))
if last_min + 1 < new_min:
# Update last_min
last_min += 1
if test_loader.dataset.set == 'validation' and last_min % 1 == 0:
#####################################
# Results on the whole validation set
#####################################
# Confusions for our subparts of validation set
Confs = np.zeros((len(predictions), nc_tot, nc_tot),
dtype=np.int32)
for i, (preds,
truth) in enumerate(zip(predictions, targets)):
# Confusions
Confs[i, :, :] = fast_confusion(
truth, preds,
test_loader.dataset.label_values).astype(np.int32)
# Show vote results
print('\nCompute confusion')
val_preds = []
val_labels = []
t1 = time.time()
for i, seq_frames in enumerate(test_loader.dataset.frames):
val_preds += [np.hstack(all_f_preds[i])]
val_labels += [np.hstack(all_f_labels[i])]
val_preds = np.hstack(val_preds)
val_labels = np.hstack(val_labels)
t2 = time.time()
C_tot = fast_confusion(val_labels, val_preds,
test_loader.dataset.label_values)
t3 = time.time()
print(' Stacking time : {:.1f}s'.format(t2 - t1))
print('Confusion time : {:.1f}s'.format(t3 - t2))
s1 = '\n'
for cc in C_tot:
for c in cc:
s1 += '{:7.0f} '.format(c)
s1 += '\n'
if debug:
print(s1)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(
list(enumerate(test_loader.dataset.label_values))):
if label_value in test_loader.dataset.ignored_labels:
C_tot = np.delete(C_tot, l_ind, axis=0)
C_tot = np.delete(C_tot, l_ind, axis=1)
# Objects IoU
val_IoUs = IoU_from_confusions(C_tot)
# Compute IoUs
mIoU = np.mean(val_IoUs)
s2 = '{:5.2f} | '.format(100 * mIoU)
for IoU in val_IoUs:
s2 += '{:5.2f} '.format(100 * IoU)
print(s2 + '\n')
# Save a report
report_file = join(
report_path,
'report_{:04d}.txt'.format(int(np.floor(last_min))))
str = 'Report of the confusion and metrics\n'
str += '***********************************\n\n\n'
str += 'Confusion matrix:\n\n'
str += s1
str += '\nIoU values:\n\n'
str += s2
str += '\n\n'
with open(report_file, 'w') as f:
f.write(str)
test_epoch += 1
# Break when reaching number of desired votes
if last_min > num_votes:
break
return
def cloud_segmentation_predict(self, net, val_loader, config, debug=False):
"""
Validation method for cloud segmentation models
"""
############
# Initialize
############
t0 = time.time()
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
val_smooth = 0.95
softmax = torch.nn.Softmax(1)
# Do not validate if dataset has no validation cloud
# if val_loader.dataset.validation_split not in val_loader.dataset.all_splits:
# return
# Number of classes including ignored labels
nc_tot = val_loader.dataset.num_classes
# Number of classes predicted by the model
nc_model = config.num_classes
#print(nc_tot)
#print(nc_model)
# Initiate global prediction over validation clouds
if not hasattr(self, 'validation_probs'):
self.validation_probs = [
np.zeros((l.shape[0], nc_model))
for l in val_loader.dataset.input_labels
]
self.val_proportions = np.zeros(nc_model, dtype=np.float32)
i = 0
for label_value in val_loader.dataset.label_values:
if label_value not in val_loader.dataset.ignored_labels:
self.val_proportions[i] = np.sum([
np.sum(labels == label_value)
for labels in val_loader.dataset.validation_labels
])
i += 1
#####################
# Network predictions
#####################
predictions = []
targets = []
t = [time.time()]
last_display = time.time()
mean_dt = np.zeros(1)
t1 = time.time()
# Start prediction loop
net.eval()
for i, batch in enumerate(val_loader):
# New time
t = t[-1:]
t += [time.time()]
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
# Get probs and labels
stacked_probs = softmax(outputs).cpu().detach().numpy()
labels = batch.labels.cpu().numpy()
lengths = batch.lengths[0].cpu().numpy()
in_inds = batch.input_inds.cpu().numpy()
cloud_inds = batch.cloud_inds.cpu().numpy()
torch.cuda.synchronize(self.device)
# Get predictions and labels per instance
# ***************************************
i0 = 0
for b_i, length in enumerate(lengths):
# Get prediction
target = labels[i0:i0 + length]
probs = stacked_probs[i0:i0 + length]
inds = in_inds[i0:i0 + length]
c_i = cloud_inds[b_i]
# Update current probs in whole cloud
self.validation_probs[c_i][inds] = val_smooth * self.validation_probs[c_i][inds] \
+ (1 - val_smooth) * probs
# Stack all prediction for this epoch
predictions.append(probs)
targets.append(target.astype(np.int64))
i0 += length
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) -
np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
print(
message.format(100 * i / config.validation_size,
1000 * (mean_dt[0]), 1000 * (mean_dt[1])))
t2 = time.time()
# Confusions for our subparts of validation set
Confs = np.zeros((len(predictions), nc_tot, nc_tot), dtype=np.int32)
for i, (probs, truth) in enumerate(zip(predictions, targets)):
# Insert false columns for ignored labels
for l_ind, label_value in enumerate(
val_loader.dataset.label_values):
if label_value in val_loader.dataset.ignored_labels:
probs = np.insert(probs, l_ind, 0, axis=1)
# Predicted labels
preds = val_loader.dataset.label_values[np.argmax(probs, axis=1)]
# Confusions
#Here
#print("Truth = {}, Predictions = {}, Label Values = {}".format(truth.dtype, preds.dtype, val_loader.dataset.label_values))
Confs[i, :, :] = fast_confusion(
truth, preds, val_loader.dataset.label_values).astype(np.int32)
t3 = time.time()
# Sum all confusions
C = np.sum(Confs, axis=0).astype(np.float32)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(
list(enumerate(val_loader.dataset.label_values))):
if label_value in val_loader.dataset.ignored_labels:
C = np.delete(C, l_ind, axis=0)
C = np.delete(C, l_ind, axis=1)
# Balance with real validation proportions
C *= np.expand_dims(self.val_proportions / (np.sum(C, axis=1) + 1e-6),
1)
t4 = time.time()
# Objects IoU
IoUs = IoU_from_confusions(C)
t5 = time.time()
# Saving (optionnal)
if config.saving:
# Name of saving file
test_file = join(config.saving_path, 'test_IoUs.txt')
print(test_file)
# Line to write:
line = ''
for IoU in IoUs:
line += '{:.3f} '.format(IoU)
line = line + '\n'
# Write in file
if exists(test_file):
with open(test_file, "a") as text_file:
text_file.write(line)
else:
with open(test_file, "w") as text_file:
text_file.write(line)
# Save potentials
pot_path = join(config.saving_path, 'potentials')
if not exists(pot_path):
makedirs(pot_path)
files = val_loader.dataset.files
for i, file_path in enumerate(files):
pot_points = np.array(val_loader.dataset.pot_trees[i].data,
copy=False)
cloud_name = file_path.split('/')[-1]
pot_name = join(pot_path, cloud_name)
pots = val_loader.dataset.potentials[i].numpy().astype(
np.float32)
write_ply(pot_name, [pot_points.astype(np.float32), pots],
['x', 'y', 'z', 'pots'])
t6 = time.time()
# Print instance mean
mIoU = 100 * np.mean(IoUs)
print('{:s} mean IoU = {:.1f}%'.format(config.dataset, mIoU))
# Save predicted cloud occasionally
if config.saving:
val_path = join(config.saving_path,
'val_preds_{:d}'.format(self.epoch + 1))
if not exists(val_path):
makedirs(val_path)
files = val_loader.dataset.files
for i, file_path in enumerate(files):
# Get points
points = val_loader.dataset.load_evaluation_points(file_path)
# Get probs on our own ply points
sub_probs = self.validation_probs[i]
# Insert false columns for ignored labels
for l_ind, label_value in enumerate(
val_loader.dataset.label_values):
if label_value in val_loader.dataset.ignored_labels:
sub_probs = np.insert(sub_probs, l_ind, 0, axis=1)
# Get the predicted labels
sub_preds = val_loader.dataset.label_values[np.argmax(
sub_probs, axis=1).astype(np.int32)]
# Reproject preds on the evaluations points
preds = (sub_preds[val_loader.dataset.test_proj[i]]).astype(
np.int32)
# Path of saved validation file
cloud_name = file_path.split('/')[-1]
val_name = join(val_path, cloud_name)
# Save file
labels = val_loader.dataset.validation_labels[i].astype(
np.int32)
write_ply(val_name, [points, preds, labels],
['x', 'y', 'z', 'preds', 'class'])
# Display timings
t7 = time.time()
net.train()
if debug:
print('\n************************\n')
print('Validation timings:')
print('Init ...... {:.1f}s'.format(t1 - t0))
print('Loop ...... {:.1f}s'.format(t2 - t1))
print('Confs ..... {:.1f}s'.format(t3 - t2))
print('Confs bis . {:.1f}s'.format(t4 - t3))
print('IoU ....... {:.1f}s'.format(t5 - t4))
print('Save1 ..... {:.1f}s'.format(t6 - t5))
print('Save2 ..... {:.1f}s'.format(t7 - t6))
print('\n************************\n')
return
def classification_test(self, net, test_loader, config, num_votes=100, debug=False):
############
# Initialize
############
# Choose test smoothing parameter (0 for no smothing, 0.99 for big smoothing)
softmax = torch.nn.Softmax(1)
# Number of classes including ignored labels
nc_tot = test_loader.dataset.num_classes
# Number of classes predicted by the model
nc_model = config.num_classes
# Initiate global prediction over test clouds
self.test_probs = np.zeros((test_loader.dataset.num_models, nc_model))
self.test_counts = np.zeros((test_loader.dataset.num_models, nc_model))
t = [time.time()]
mean_dt = np.zeros(1)
last_display = time.time()
while np.min(self.test_counts) < num_votes:
# Run model on all test examples
# ******************************
# Initiate result containers
probs = []
targets = []
obj_inds = []
# Start validation loop
for batch in test_loader:
# New time
t = t[-1:]
t += [time.time()]
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
# Get probs and labels
probs += [softmax(outputs).cpu().detach().numpy()]
targets += [batch.labels.cpu().numpy()]
obj_inds += [batch.model_inds.cpu().numpy()]
if 'cuda' in self.device.type:
torch.cuda.synchronize(self.device)
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) - np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'Test vote {:.0f} : {:.1f}% (timings : {:4.2f} {:4.2f})'
print(message.format(np.min(self.test_counts),
100 * len(obj_inds) / config.validation_size,
1000 * (mean_dt[0]),
1000 * (mean_dt[1])))
# Stack all validation predictions
probs = np.vstack(probs)
targets = np.hstack(targets)
obj_inds = np.hstack(obj_inds)
if np.any(test_loader.dataset.input_labels[obj_inds] != targets):
raise ValueError('wrong object indices')
# Compute incremental average (predictions are always ordered)
self.test_counts[obj_inds] += 1
self.test_probs[obj_inds] += (probs - self.test_probs[obj_inds]) / (self.test_counts[obj_inds])
# Save/Display temporary results
# ******************************
test_labels = np.array(test_loader.dataset.label_values)
# Compute classification results
C1 = fast_confusion(test_loader.dataset.input_labels,
np.argmax(self.test_probs, axis=1),
test_labels)
ACC = 100 * np.sum(np.diag(C1)) / (np.sum(C1) + 1e-6)
print('Test Accuracy = {:.1f}%'.format(ACC))
return
def slam_segmentation_validation(self,
net,
val_loader,
config,
debug=True):
"""
Validation method for slam segmentation models
"""
############
# Initialize
############
t0 = time.time()
# Do not validate if dataset has no validation cloud
if val_loader is None:
return
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
val_smooth = 0.95
softmax = torch.nn.Softmax(1)
# Create folder for validation predictions
if not exists(join(config.saving_path, 'val_preds')):
makedirs(join(config.saving_path, 'val_preds'))
# initiate the dataset validation containers
val_loader.dataset.val_points = []
val_loader.dataset.val_labels = []
# Number of classes including ignored labels
nc_tot = val_loader.dataset.num_classes
#####################
# Network predictions
#####################
predictions = []
targets = []
inds = []
val_i = 0
t = [time.time()]
last_display = time.time()
mean_dt = np.zeros(1)
t1 = time.time()
# Start validation loop
for i, batch in enumerate(val_loader):
# New time
t = t[-1:]
t += [time.time()]
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
# Get probs and labels
stk_probs = softmax(outputs).cpu().detach().numpy()
lengths = batch.lengths[0].cpu().numpy()
f_inds = batch.frame_inds.cpu().numpy()
r_inds_list = batch.reproj_inds
r_mask_list = batch.reproj_masks
labels_list = batch.val_labels
torch.cuda.synchronize(self.device)
# Get predictions and labels per instance
# ***************************************
i0 = 0
for b_i, length in enumerate(lengths):
# Get prediction
probs = stk_probs[i0:i0 + length]
proj_inds = r_inds_list[b_i]
proj_mask = r_mask_list[b_i]
frame_labels = labels_list[b_i]
s_ind = f_inds[b_i, 0]
f_ind = f_inds[b_i, 1]
# Project predictions on the frame points
proj_probs = probs[proj_inds]
# Safe check if only one point:
if proj_probs.ndim < 2:
proj_probs = np.expand_dims(proj_probs, 0)
# Insert false columns for ignored labels
for l_ind, label_value in enumerate(
val_loader.dataset.label_values):
if label_value in val_loader.dataset.ignored_labels:
proj_probs = np.insert(proj_probs, l_ind, 0, axis=1)
# Predicted labels
preds = val_loader.dataset.label_values[np.argmax(proj_probs,
axis=1)]
# Save predictions in a binary file
filename = '{:s}_{:07d}.npy'.format(
val_loader.dataset.sequences[s_ind], f_ind)
filepath = join(config.saving_path, 'val_preds', filename)
if exists(filepath):
frame_preds = np.load(filepath)
else:
frame_preds = np.zeros(frame_labels.shape, dtype=np.uint8)
frame_preds[proj_mask] = preds.astype(np.uint8)
np.save(filepath, frame_preds)
# Save some of the frame pots
if f_ind % 20 == 0:
seq_path = join(val_loader.dataset.path, 'sequences',
val_loader.dataset.sequences[s_ind])
velo_file = join(
seq_path, 'velodyne',
val_loader.dataset.frames[s_ind][f_ind] + '.bin')
frame_points = np.fromfile(velo_file, dtype=np.float32)
frame_points = frame_points.reshape((-1, 4))
write_ply(filepath[:-4] + '_pots.ply',
[frame_points[:, :3], frame_labels, frame_preds],
['x', 'y', 'z', 'gt', 'pre'])
# Update validation confusions
frame_C = fast_confusion(frame_labels,
frame_preds.astype(np.int32),
val_loader.dataset.label_values)
val_loader.dataset.val_confs[s_ind][f_ind, :, :] = frame_C
# Stack all prediction for this epoch
predictions += [preds]
targets += [frame_labels[proj_mask]]
inds += [f_inds[b_i, :]]
val_i += 1
i0 += length
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) -
np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
print(
message.format(100 * i / config.validation_size,
1000 * (mean_dt[0]), 1000 * (mean_dt[1])))
t2 = time.time()
# Confusions for our subparts of validation set
Confs = np.zeros((len(predictions), nc_tot, nc_tot), dtype=np.int32)
for i, (preds, truth) in enumerate(zip(predictions, targets)):
# Confusions
Confs[i, :, :] = fast_confusion(
truth, preds, val_loader.dataset.label_values).astype(np.int32)
t3 = time.time()
#######################################
# Results on this subpart of validation
#######################################
# Sum all confusions
C = np.sum(Confs, axis=0).astype(np.float32)
# Balance with real validation proportions
C *= np.expand_dims(
val_loader.dataset.class_proportions / (np.sum(C, axis=1) + 1e-6),
1)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(
list(enumerate(val_loader.dataset.label_values))):
if label_value in val_loader.dataset.ignored_labels:
C = np.delete(C, l_ind, axis=0)
C = np.delete(C, l_ind, axis=1)
# Objects IoU
IoUs = IoU_from_confusions(C)
#####################################
# Results on the whole validation set
#####################################
t4 = time.time()
# Sum all validation confusions
C_tot = [
np.sum(seq_C, axis=0) for seq_C in val_loader.dataset.val_confs
if len(seq_C) > 0
]
C_tot = np.sum(np.stack(C_tot, axis=0), axis=0)
if debug:
s = '\n'
for cc in C_tot:
for c in cc:
s += '{:8.1f} '.format(c)
s += '\n'
print(s)
# Remove ignored labels from confusions
for l_ind, label_value in reversed(
list(enumerate(val_loader.dataset.label_values))):
if label_value in val_loader.dataset.ignored_labels:
C_tot = np.delete(C_tot, l_ind, axis=0)
C_tot = np.delete(C_tot, l_ind, axis=1)
# Objects IoU
val_IoUs = IoU_from_confusions(C_tot)
t5 = time.time()
# Saving (optionnal)
if config.saving:
IoU_list = [IoUs, val_IoUs]
file_list = ['subpart_IoUs.txt', 'val_IoUs.txt']
for IoUs_to_save, IoU_file in zip(IoU_list, file_list):
# Name of saving file
test_file = join(config.saving_path, IoU_file)
# Line to write:
line = ''
for IoU in IoUs_to_save:
line += '{:.3f} '.format(IoU)
line = line + '\n'
# Write in file
if exists(test_file):
with open(test_file, "a") as text_file:
text_file.write(line)
else:
with open(test_file, "w") as text_file:
text_file.write(line)
# Print instance mean
mIoU = 100 * np.mean(IoUs)
print('{:s} : subpart mIoU = {:.1f} %'.format(config.dataset, mIoU))
mIoU = 100 * np.mean(val_IoUs)
print('{:s} : val mIoU = {:.1f} %'.format(config.dataset, mIoU))
t6 = time.time()
# Display timings
if debug:
print('\n************************\n')
print('Validation timings:')
print('Init ...... {:.1f}s'.format(t1 - t0))
print('Loop ...... {:.1f}s'.format(t2 - t1))
print('Confs ..... {:.1f}s'.format(t3 - t2))
print('IoU1 ...... {:.1f}s'.format(t4 - t3))
print('IoU2 ...... {:.1f}s'.format(t5 - t4))
print('Save ...... {:.1f}s'.format(t6 - t5))
print('\n************************\n')
return
def object_classification_validation(self, net, val_loader, config):
"""
Perform a round of validation and show/save results
:param net: network object
:param val_loader: data loader for validation set
:param config: configuration object
"""
############
# Initialize
############
# Choose validation smoothing parameter (0 for no smothing, 0.99 for big smoothing)
val_smooth = 0.95
# Number of classes predicted by the model
nc_model = config.num_classes
softmax = torch.nn.Softmax(1)
# Initialize global prediction over all models
if not hasattr(self, 'val_probs'):
self.val_probs = np.zeros(
(val_loader.dataset.num_models, nc_model))
#####################
# Network predictions
#####################
probs = []
targets = []
obj_inds = []
t = [time.time()]
last_display = time.time()
mean_dt = np.zeros(1)
# Start validation loop
for batch in val_loader:
# New time
t = t[-1:]
t += [time.time()]
if 'cuda' in self.device.type:
batch.to(self.device)
# Forward pass
outputs = net(batch, config)
# Get probs and labels
probs += [softmax(outputs).cpu().detach().numpy()]
targets += [batch.labels.cpu().numpy()]
obj_inds += [batch.model_inds.cpu().numpy()]
torch.cuda.synchronize(self.device)
# Average timing
t += [time.time()]
mean_dt = 0.95 * mean_dt + 0.05 * (np.array(t[1:]) -
np.array(t[:-1]))
# Display
if (t[-1] - last_display) > 1.0:
last_display = t[-1]
message = 'Validation : {:.1f}% (timings : {:4.2f} {:4.2f})'
print(
message.format(
100 * len(obj_inds) / config.validation_size,
1000 * (mean_dt[0]), 1000 * (mean_dt[1])))
# Stack all validation predictions
probs = np.vstack(probs)
targets = np.hstack(targets)
obj_inds = np.hstack(obj_inds)
###################
# Voting validation
###################
self.val_probs[obj_inds] = val_smooth * self.val_probs[obj_inds] + (
1 - val_smooth) * probs
############
# Confusions
############
validation_labels = np.array(val_loader.dataset.label_values)
# Compute classification results
C1 = fast_confusion(targets, np.argmax(probs, axis=1),
validation_labels)
# Compute votes confusion
C2 = fast_confusion(val_loader.dataset.input_labels,
np.argmax(self.val_probs, axis=1),
validation_labels)
# Saving (optionnal)
if config.saving:
print("Save confusions")
conf_list = [C1, C2]
file_list = ['val_confs.txt', 'vote_confs.txt']
for conf, conf_file in zip(conf_list, file_list):
test_file = join(config.saving_path, conf_file)
if exists(test_file):
with open(test_file, "a") as text_file:
for line in conf:
for value in line:
text_file.write('%d ' % value)
text_file.write('\n')
else:
with open(test_file, "w") as text_file:
for line in conf:
for value in line:
text_file.write('%d ' % value)
text_file.write('\n')
val_ACC = 100 * np.sum(np.diag(C1)) / (np.sum(C1) + 1e-6)
vote_ACC = 100 * np.sum(np.diag(C2)) / (np.sum(C2) + 1e-6)
print('Accuracies : val = {:.1f}% / vote = {:.1f}%'.format(
val_ACC, vote_ACC))
return C1