def get_least_accurate_sample_using_labels(self, model, images,
selection_count):
model.eval()
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
num_inaccurate_pixels = []
with torch.no_grad():
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
output = model(image_batch)
prediction = torch.argmax(output,
dim=1).type(torch.cuda.FloatTensor)
for idx in range(prediction.shape[0]):
mask = (label_batch[idx, :, :] >=
0) & (label_batch[idx, :, :] < self.num_classes)
incorrect = label_batch[idx, mask] != prediction[idx, mask]
num_inaccurate_pixels.append(
incorrect.sum().cpu().float().item())
selected_samples = list(
zip(*sorted(zip(num_inaccurate_pixels, images),
key=lambda x: x[0],
reverse=True)))[1][:selection_count]
return selected_samples
def get_unsure_samples(self, model, images, selection_count):
model.eval()
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
softmax = torch.nn.Softmax2d()
scores = []
with torch.no_grad():
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
deeplab_output, unet_output = model(image_batch)
prediction = softmax(unet_output)
for idx in range(prediction.shape[0]):
mask = (label_batch[idx, :, :] >=
0) & (label_batch[idx, :, :] < self.num_classes)
y = 4 * prediction[idx, 1, mask] - 4 * prediction[idx, 1,
mask]**2
scores.append(y.mean().cpu().float().item())
selected_samples = list(
zip(*sorted(zip(scores, images), key=lambda x: x[0],
reverse=True)))[1][:selection_count]
print(scores)
return selected_samples
def get_vote_entropy_for_images_with_input_noise(self, model, images,
selection_count):
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
model.eval()
entropies = []
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
entropies.extend([
torch.sum(x).cpu().item() /
(image_batch.shape[2] * image_batch.shape[3])
for x in self._get_vote_entropy_for_batch_with_input_noise(
model, image_batch, label_batch)
])
selected_samples = list(
zip(*sorted(zip(entropies, images),
key=lambda x: x[0],
reverse=True)))[1][:selection_count]
return selected_samples
def get_vote_entropy_for_images(self, model, images, selection_count):
def turn_on_dropout(m):
if type(m) == torch.nn.Dropout2d:
m.train()
model.apply(turn_on_dropout)
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
entropies = []
#times = []
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
#a = time.time()
entropies.extend([
torch.mean(x).cpu().item()
for x in self._get_vote_entropy_for_batch(
model, image_batch, label_batch)
])
#times.append(time.time() - a)
#print(np.mean(times), np.std(times))
model.eval()
selected_samples = list(
zip(*sorted(zip(entropies, images),
key=lambda x: x[0],
reverse=True)))[1][:selection_count]
return selected_samples
def get_least_accurate_samples(self,
model,
images,
selection_count,
mode='softmax'):
model.eval()
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
num_inaccurate_pixels = []
softmax = torch.nn.Softmax2d()
#times = []
with torch.no_grad():
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
#a = time.time()
deeplab_output, unet_output = model(image_batch)
if mode == 'softmax':
prediction = softmax(unet_output)
for idx in range(prediction.shape[0]):
mask = (label_batch[idx, :, :] >= 0) & (
label_batch[idx, :, :] < self.num_classes)
incorrect = prediction[idx, 0, mask]
num_inaccurate_pixels.append(
incorrect.sum().cpu().float().item())
elif mode == 'argmax':
prediction = unet_output.argmax(1).squeeze().type(
torch.cuda.FloatTensor)
for idx in range(prediction.shape[0]):
mask = (label_batch[idx, :, :] >= 0) & (
label_batch[idx, :, :] < self.num_classes)
incorrect = 1 - prediction[idx, mask]
num_inaccurate_pixels.append(
incorrect.sum().cpu().float().item())
else:
raise NotImplementedError
#times.append(time.time() - a)
#print(np.mean(times), np.std(times))
selected_samples = list(
zip(*sorted(zip(num_inaccurate_pixels, images),
key=lambda x: x[0],
reverse=True)))[1][:selection_count]
return selected_samples
def get_k_center_greedy_selections(self, selection_size, model,
candidate_image_batch,
already_selected_image_batch):
combined_paths = already_selected_image_batch + candidate_image_batch
loader = DataLoader(paths_dataset.PathsDataset(self.env,
combined_paths,
self.crop_size),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
if model.module.model_name == 'deeplab':
FEATURE_DIM = 2736
average_pool_kernel_size = (64, 64)
elif model.module.model_name == 'enet':
FEATURE_DIM = 1152
average_pool_kernel_size = (32, 32)
features = np.zeros((len(combined_paths), FEATURE_DIM))
model.eval()
model.module.set_return_features(True)
#times = []
average_pool_stride = average_pool_kernel_size[0] // 2
with torch.no_grad():
for batch_idx, sample in enumerate(tqdm(loader)):
#a = time.time()
_, features_batch = model(sample.cuda())
features_batch = F.avg_pool2d(features_batch,
average_pool_kernel_size,
average_pool_stride)
for feature_idx in range(features_batch.shape[0]):
features[batch_idx * self.dataloader_batch_size +
feature_idx, :] = features_batch[
feature_idx, :, :, :].cpu().numpy().flatten()
#times.append(time.time() - a)
#print(np.mean(times), np.std(times))
model.module.set_return_features(False)
selected_indices = self._select_batch(
features, list(range(len(already_selected_image_batch))),
selection_size)
return [combined_paths[i] for i in selected_indices]
def get_least_margin_samples(self, model, images, selection_count):
model.eval()
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
margins = []
with torch.no_grad():
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
softmax = torch.nn.Softmax2d()
output = softmax(model(image_batch))
for batch_idx in range(output.shape[0]):
mask = (label_batch[batch_idx, :, :] <
0) | (label_batch[batch_idx, :, :] >=
self.dataset_num_classes)
mask = mask.cpu().numpy().astype(np.bool)
most_confident_scores = torch.max(
output[batch_idx, :, :].squeeze(),
dim=0)[0].cpu().numpy()
output_numpy = output[batch_idx, :, :, :].cpu().numpy()
ndx = np.indices(output_numpy.shape)
second_most_confident_scores = output_numpy[
output_numpy.argsort(0), ndx[1], ndx[2]][-2]
margin = most_confident_scores - second_most_confident_scores
margin[mask] = 1
# from active_selection import ActiveSelectionMCDropout
# prediction = np.argmax(output[batch_idx, :, :, :].cpu().numpy().squeeze(), axis=0)
# ActiveSelectionMCDropout._visualize_entropy(image_batch[batch_idx, :, :, :].cpu().numpy(), margin, prediction)
margins.append(np.mean(margin))
selected_samples = list(
zip(*sorted(zip(margins, images),
key=lambda x: x[0],
reverse=False)))[1][:selection_count]
return selected_samples
def get_weakly_labeled_data(self,
model,
images,
threshold,
entropies=None):
if not entropies:
entropies = self._get_entropies(model, images)
selected_images = []
weak_labels = []
for image, entropy in zip(images, entropies):
if entropy < threshold:
selected_images.append(image)
loader = DataLoader(paths_dataset.PathsDataset(self.env,
selected_images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
with torch.no_grad():
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
output = model(image_batch)
for batch_idx in range(output.shape[0]):
mask = (label_batch[batch_idx, :, :] <
0) | (label_batch[batch_idx, :, :] >=
self.dataset_num_classes)
mask = mask.cpu().numpy().astype(np.bool)
prediction = np.argmax(
output[batch_idx, :, :, :].cpu().numpy().squeeze(),
axis=0).astype(np.uint8)
prediction[mask] = 255
weak_labels.append(prediction)
return dict(zip(selected_images, weak_labels))
def _get_entropies(self, model, images):
model.eval()
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
entropies = []
#times = []
with torch.no_grad():
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
#a = time.time()
softmax = torch.nn.Softmax2d()
output = softmax(model(image_batch))
num_classes = output.shape[1]
for batch_idx in range(output.shape[0]):
mask = (label_batch[batch_idx, :, :] <
0) | (label_batch[batch_idx, :, :] >=
self.dataset_num_classes)
entropy_map = torch.cuda.FloatTensor(
output.shape[2], output.shape[3]).fill_(0)
for c in range(num_classes):
entropy_map = entropy_map - (
output[batch_idx, c, :, :] *
torch.log2(output[batch_idx, c, :, :] + 1e-12))
entropy_map[mask] = 0
# from active_selection import ActiveSelectionMCDropout
# prediction = np.argmax(output[batch_idx, :, :, :].cpu().numpy().squeeze(), axis=0)
# ActiveSelectionMCDropout._visualize_entropy(image_batch[batch_idx, :, :, :].cpu().numpy(), entropy_map.cpu().numpy(), prediction)
entropies.append(np.mean(entropy_map.cpu().numpy()))
#times.append(time.time() - a)
##print(np.mean(times), np.std(times))
return entropies
def get_adversarially_vulnarable_samples(self, model, images,
selection_count):
model.eval()
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
softmax = torch.nn.Softmax2d()
scores = []
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
with torch.no_grad():
deeplab_output, unet_output = model(image_batch)
prediction = softmax(unet_output)
unet_input = torch.cuda.FloatTensor(
torch.cat([softmax(deeplab_output), image_batch],
dim=1).detach().cpu().numpy())
unet_input.requires_grad = True
only_unet_output = model.module.unet(unet_input)
only_unet_output.backward(torch.ones_like(only_unet_output).cuda())
gradient_norms = torch.norm(unet_input.grad, p=2, dim=1)
for idx in range(prediction.shape[0]):
mask = (label_batch[idx, :, :] < 0) | (label_batch[idx, :, :]
>= self.num_classes)
gradient_norms[idx, mask] = 0
scores.append(
gradient_norms[idx, :, :].mean().cpu().float().item())
selected_samples = list(
zip(*sorted(zip(scores, images), key=lambda x: x[0],
reverse=True)))[1][:selection_count]
return selected_samples
def get_least_accurate_region_maps(self, model, images, existing_regions,
region_size, selection_size):
base_size = 512 if self.crop_size == -1 else self.crop_size
score_maps = torch.cuda.FloatTensor(len(images),
base_size - region_size + 1,
base_size - region_size + 1)
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
weights = torch.cuda.FloatTensor(region_size, region_size).fill_(1.)
map_ctr = 0
#times = []
# commented lines are for visualization and verification
#error_maps = []
#base_images = []
softmax = torch.nn.Softmax2d()
with torch.no_grad():
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
#a = time.time()
deeplab_output, unet_output = model(image_batch)
prediction = softmax(unet_output)
for idx in range(prediction.shape[0]):
mask = (label_batch[idx, :, :] <
0) | (label_batch[idx, :, :] >= self.num_classes)
incorrect = prediction[idx, 0, :, :]
incorrect[mask] = 0
self.suppress_labeled_areas(incorrect,
existing_regions[map_ctr])
#base_images.append(image_batch[idx, :, :, :].cpu().numpy())
# error_maps.append(incorrect.cpu().numpy())
score_maps[map_ctr, :, :] = torch.nn.functional.conv2d(
incorrect.unsqueeze(0).unsqueeze(0),
weights.unsqueeze(0).unsqueeze(0)).squeeze().squeeze()
map_ctr += 1
#times.append(time.time() - a)
min_val = score_maps.min()
max_val = score_maps.max()
minmax_norm = lambda x: x.add_(-min_val).mul_(1.0 /
(max_val - min_val))
minmax_norm(score_maps)
num_requested_indices = (selection_size * base_size *
base_size) / (region_size * region_size)
#print(np.mean(times), np.std(times))
regions, num_selected_indices = ActiveSelectionMCDropout.square_nms(
score_maps.cpu(), region_size, num_requested_indices)
# print(f'Requested/Selected indices {num_requested_indices}/{num_selected_indices}')
# for i in range(len(regions)):
# ActiveSelectionMCDropout._visualize_regions(base_images[i], error_maps[i], regions[i], score_maps[i, :, :].cpu().numpy(), region_size)
new_regions = {}
for i in range(len(regions)):
if regions[i] != []:
new_regions[images[i]] = regions[i]
model.eval()
return new_regions, num_selected_indices
def get_least_confident_samples(self, model, images, selection_count):
model.eval()
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
max_confidence = []
#rgb_images = []
#sem_gt_images = []
#sem_pred_images = []
#lc_images = []
with torch.no_grad():
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
softmax = torch.nn.Softmax2d()
output = model(image_batch)
max_conf_batch = torch.max(softmax(output), dim=1)[0]
for batch_idx in range(max_conf_batch.shape[0]):
mask = (label_batch[batch_idx, :, :] <
0) | (label_batch[batch_idx, :, :] >=
self.dataset_num_classes)
max_conf_batch[batch_idx, mask] = 1
# from active_selection import ActiveSelectionMCDropout
# ActiveSelectionMCDropout._visualize_entropy(image_batch[batch_idx, :, :, :].cpu().numpy(), max_conf_batch[
# batch_idx, :, :].cpu().numpy(), prediction)
'''
image_unnormalized = ((np.transpose(image_batch[batch_idx].cpu().numpy(), axes=[1, 2, 0])
* (0.229, 0.224, 0.225) + (0.485, 0.456, 0.406)) * 255).astype(np.uint8)
rgb_images.append(image_unnormalized)
gt_colored = map_segmentation_to_colors(np.array(label_batch[batch_idx].cpu().numpy()).astype(np.uint8), 'cityscapes')
sem_gt_images.append(gt_colored)
prediction = np.argmax(output[batch_idx, :, :, :].cpu().numpy().squeeze(), axis=0)
sem_pred_images.append(map_segmentation_to_colors(np.array(prediction).astype(np.uint8), 'cityscapes'))
masked_target_array = np.ma.array(max_conf_batch[batch_idx, :, :].cpu().numpy(), mask=label_batch[batch_idx].cpu().numpy() == 255)
masked_target_array = 1 - masked_target_array
cmap = matplotlib.cm.jet
cmap.set_bad('white', 1.)
lc_images.append(cmap(masked_target_array))
'''
max_confidence.append(
torch.mean(
max_conf_batch[batch_idx, :, :]).cpu().item())
'''
import matplotlib.pyplot as plt
for prefix, arr in zip(['rgb', 'sem_gt', 'sem_pred', 'lc'], [rgb_images, sem_gt_images, sem_pred_images, lc_images]):
stacked_image = np.ones(((arr[0].shape[0] + 20) * len(arr), arr[0].shape[1], arr[0].shape[2]),
dtype=arr[0].dtype) * (255 if arr[0].dtype == np.uint8 else 1)
for i, im in enumerate(arr):
stacked_image[i * (arr[0].shape[0] + 20): i * (arr[0].shape[0] + 20) + arr[0].shape[0], :, :] = im
plt.imsave('%s.png' % (prefix), stacked_image)
'''
selected_samples = list(
zip(*sorted(zip(max_confidence, images),
key=lambda x: x[0],
reverse=False)))[1][:selection_count]
return selected_samples
def create_region_maps(self, model, images, existing_regions, region_size,
selection_size):
base_size = 512 if self.crop_size == -1 else self.crop_size
score_maps = torch.cuda.FloatTensor(len(images),
base_size - region_size + 1,
base_size - region_size + 1)
loader = DataLoader(paths_dataset.PathsDataset(self.env,
images,
self.crop_size,
include_labels=True),
batch_size=self.dataloader_batch_size,
shuffle=False,
num_workers=0)
weights = torch.cuda.FloatTensor(region_size, region_size).fill_(1.)
map_ctr = 0
# commented lines are for visualization and verification
# entropy_maps = []
# base_images = []
for sample in tqdm(loader):
image_batch = sample['image'].cuda()
label_batch = sample['label'].cuda()
noise_entropies = self._get_vote_entropy_for_batch_with_feature_noise(
model, image_batch, label_batch)
mc_entropies = self._get_vote_entropy_for_batch_with_mc_dropout(
model, image_batch, label_batch)
combined_entropies = [
x + y for x, y in zip(noise_entropies, mc_entropies)
]
for img_idx, entropy_map in enumerate(combined_entropies):
ActiveSelectionMCDropout.suppress_labeled_entropy(
entropy_map, existing_regions[map_ctr])
# base_images.append(image_batch[img_idx, :, :, :].cpu().numpy())
# entropy_maps.append(entropy_map.cpu().numpy())
score_maps[map_ctr, :, :] = torch.nn.functional.conv2d(
entropy_map.unsqueeze(0).unsqueeze(0),
weights.unsqueeze(0).unsqueeze(0)).squeeze().squeeze()
map_ctr += 1
min_val = score_maps.min()
max_val = score_maps.max()
minmax_norm = lambda x: x.add_(-min_val).mul_(1.0 /
(max_val - min_val))
minmax_norm(score_maps)
num_requested_indices = (selection_size * base_size *
base_size) / (region_size * region_size)
regions, num_selected_indices = ActiveSelectionMCDropout.square_nms(
score_maps.cpu(), region_size, num_requested_indices)
# print(f'Requested/Selected indices {num_requested_indices}/{num_selected_indices}')
# for i in range(len(regions)):
# ActiveSelectionMCDropout._visualize_regions(base_images[i], entropy_maps[i], regions[i], region_size)
new_regions = {}
for i in range(len(regions)):
if regions[i] != []:
new_regions[images[i]] = regions[i]
model.eval()
return new_regions, num_selected_indices