def get_train_dataloader(self, data_args) -> DataLoader:
self.mode = 'training'
# Deepcopy ensures any changes to mode variable will not influence this dataloader
return torch.utils.data.DataLoader(deepcopy(self),
batch_size=data_args['batch_size'],
shuffle=data_args['shuffle'],
num_workers=get_cores_count())
def get_test_dataloader(self, data_args):
self.image_dataset.mode = 'test'
self.attribution_dataset.mode = 'test'
return torch.utils.data.DataLoader(deepcopy(self),
batch_size=data_args['batch_size'],
shuffle=data_args['shuffle'],
num_workers=get_cores_count())
def test_dataloader(self):
return torch.utils.data.DataLoader(
self.testset,
batch_size=self.val_data_args['batch_size'],
shuffle=self.val_data_args['shuffle'],
pin_memory=True,
num_workers=get_cores_count())
def validation_dataloader(self) -> DataLoader:
return torch.utils.data.DataLoader(
self.validationset,
batch_size=self.train_data_args['batch_size'],
shuffle=self.train_data_args['shuffle'],
pin_memory=True,
num_workers=get_cores_count())
def __init__(
self,
dataset_args,
train_data_args,
val_data_args,
device="cuda:1",
):
"""
use_random_flip not used.
"""
torch.cuda.set_device(device)
self.cpu_count = get_cores_count()
self.train_data_args = train_data_args
self.val_data_args = val_data_args
dataset_dir = dataset_args['dataset_dir']
split_ratio = dataset_args.get('split_ratio', 7.0 / 8.0)
assert split_ratio < 1.0, 'CIFAR train set should be split into train and cross-validation set.'
# Use augmentations for training models but not during generating dataset.
self.train_transform = CIFAR10.get_train_transform(
enable_augmentation=train_data_args.get('enable_augmentation',
False))
self.validation_transform = CIFAR10.get_validation_transform()
# Normalization transform does (x - mean) / std
# To denormalize use mean* = (-mean/std) and std* = (1/std)
self.demean = [-m / s for m, s in zip(self.mean, self.std)]
self.destd = [1 / s for s in self.std]
self.denormalization_transform = torchvision.transforms.Normalize(
self.demean, self.destd, inplace=False)
self.trainset = torchvision.datasets.CIFAR10(
root=dataset_dir,
train=True,
download=True,
transform=self.train_transform)
self.validationset = torchvision.datasets.CIFAR10(
root=dataset_dir,
train=True,
download=True,
transform=self.validation_transform)
# Split train data into training and cross validation dataset using 9:1 split ration
training_indices, validation_indices = self._uniform_train_val_split(
self.trainset.targets, split_ratio)
self.trainset = torch.utils.data.Subset(self.trainset,
training_indices)
self.validationset = torch.utils.data.Subset(self.validationset,
validation_indices)
self.testset = torchvision.datasets.CIFAR10(
root=dataset_dir,
train=False,
download=True,
transform=self.validation_transform)
def perform_perturbation_analysis(arguments):
"""
"""
val_data_args = dict(batch_size=1, shuffle=False)
""" Setup result directory """
outdir = arguments['outdir']
os.makedirs(outdir, exist_ok=True)
print('Arguments:\n{}'.format(pformat(arguments)))
""" Set random seed throughout python"""
random_seed = random.randint(0, 1000)
utils.set_random_seed(random_seed=random_seed)
""" Set device - cpu or gpu """
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(f'Using device - {device}')
""" Load Model with weights(if available) """
dataset_args = arguments['data']
model_args = arguments['pixel_perturbation_analysis']['model']
model: torch.nn.Module = models_utils.get_model(model_args, device,
dataset_args).to(device)
""" Load parameters for the Dataset """
if dataset_args['dataset'] == 'ImageNet':
transform = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
testset = datasets.ImageNet(dataset_args['dataset_dir'],
split='val',
transform=transform)
else:
# ToDo - Make uniform api for loading different datasets. Birdsnap/Imagenet/Food101 supports needs to be added.
dataset = create_dataset(
dataset_args,
val_data_args, # Just use val_data_args as train_data_args
val_data_args) # Split doesnt matter, we use test dataset
testset = dataset.testset
num_samples = min(arguments['pixel_perturbation_analysis']['test_samples'],
len(testset))
print(
f'Test dataset has {len(testset)} samples. We are using randomly selected {num_samples} samples for testing.'
)
testset = torch.utils.data.Subset(
testset, random.sample(range(0, len(testset)), num_samples))
dataloader = torch.utils.data.DataLoader(testset,
batch_size=1,
shuffle=False,
num_workers=get_cores_count())
# Attribution method and percentiles at which to test.
attribution_methods = arguments['pixel_perturbation_analysis'][
'attribution_methods']
print('Running pixel perturbation analysis for: ', attribution_methods)
# Step sizes to remove top k or bottom k
percentiles = arguments['pixel_perturbation_analysis']['percentiles']
# Save plots in outdir in outdir/DATASET_[train/test]/MODEL_ATTRIBUTIONMETHOD/ImageIndex.png
model.eval()
timestamp = datetime.datetime.now().isoformat()
# To save sum of delta output change for each attribution method, percentile and
# remove top and bottom percentile pixels.
# ToDo - Use a numpy dictionary with key attribution names.
# Pros - Easy plotting. Load npy files without need to remember which index mapped to which attribution method.
output_deviation_sum = np.zeros(
(len(attribution_methods), len(percentiles), 2), dtype=float)
# Save results in corresponding directory
attribution_output_dir = os.path.join(outdir,
timestamp) # E.g. outdir/timestamp/
os.makedirs(attribution_output_dir, exist_ok=True)
for counter, data in enumerate(tqdm(dataloader, total=num_samples)):
if counter == num_samples:
break
inputs, labels = data
inputs = inputs.to(device)
outputs = model(inputs).detach().cpu()
_, max_prob_indices = torch.max(outputs.data, 1)
outputs = torch.nn.functional.softmax(outputs, dim=1)
outputs = outputs.numpy()
for attribution_method_index, attribution_method in enumerate(
attribution_methods):
for preprocessed_image, max_prob_index, output in zip(
inputs, max_prob_indices, outputs):
attribution_map = attribution_loader.generate_attribution(
model, preprocessed_image.unsqueeze(0),
max_prob_index.to(device), attribution_method)
# To take absolute value for each pixel channel for each attribution method.
attribution_map = np.max(attribution_map, axis=0)
preprocessed_image = preprocessed_image.cpu().numpy()
modified_images_bottom_remove = remove(
preprocessed_image.copy(),
attribution_map,
replace_value=[0, 0, 0],
# Black in original image is -mean/std in preprocessed image
percentiles=percentiles,
bottom=True,
gray=True)
modified_images_top_remove = remove(
preprocessed_image.copy(),
attribution_map,
replace_value=[0, 0, 0],
# Black in original image is -mean/std in preprocessed image
percentiles=percentiles,
bottom=False,
gray=True)
# Create a batch of all images
modified_images_top_remove = torch.from_numpy(
np.stack(modified_images_top_remove, axis=0)).to(device)
modified_images_bottom_remove = torch.from_numpy(
np.stack(modified_images_bottom_remove, axis=0)).to(device)
# Run forward pass - ToDo - Do in single pass
output_top_q = model(modified_images_top_remove)
output_bottom_q = model(modified_images_bottom_remove)
output_top_q = torch.nn.functional.softmax(output_top_q, dim=1)
output_bottom_q = torch.nn.functional.softmax(output_bottom_q,
dim=1)
output_top_q = output_top_q.detach().cpu().numpy()
output_bottom_q = output_bottom_q.detach().cpu().numpy()
# Get output value at max_prob_index for each percentile
output_top_q_max_class_prob = output_top_q[:, max_prob_index]
output_bottom_q_max_class_prob = output_bottom_q[:,
max_prob_index]
# Compute deviation from model output for original image at max_prob_index
top_deviation = np.abs(
(output[max_prob_index] - output_top_q_max_class_prob) /
output[max_prob_index])
bottom_deviation = np.abs(
(output[max_prob_index] - output_bottom_q_max_class_prob) /
output[max_prob_index])
# Add this deviation to right dimension of matrix
output_deviation_sum[attribution_method_index, :,
0] += top_deviation
output_deviation_sum[attribution_method_index, :,
1] += bottom_deviation
if counter % 500 == 499:
# Divide output_deviation_sum each element by num_samples
output_deviation_mean = output_deviation_sum * 100.0 / (counter +
1)
print("\nAffect of removal of most important pixels at:-")
for attribution_method_index, attribution_method in enumerate(
attribution_methods):
with np.printoptions(precision=3,
formatter={'float': '{: 0.3f}'.format},
suppress=True,
linewidth=np.inf):
print(
attribution_method['name'].ljust(20) + ' = ',
np.array2string(
output_deviation_mean[attribution_method_index, :,
0],
separator=', '))
print("Affect of removal of least important pixels at:-")
for attribution_method_index, attribution_method in enumerate(
attribution_methods):
with np.printoptions(precision=3,
formatter={'float': '{: 0.3f}'.format},
suppress=True,
linewidth=np.inf):
print(
attribution_method['name'].ljust(20) + ' = ',
np.array2string(
output_deviation_mean[attribution_method_index, :,
1],
separator=', '))
print()
# Divide output_deviation_sum each element by num_samples
output_deviation_mean = output_deviation_sum * 100.0 / num_samples
# Save in directory
np.save(os.path.join(attribution_output_dir, 'pixel_perturbation.npy'),
output_deviation_mean)
with np.printoptions(precision=3,
formatter={'float': '{: 0.3f}'.format},
suppress=True,
linewidth=np.inf):
print("Affect of removal of most important pixels at:- \npercentiles ",
percentiles)
for ind, attr in enumerate(attribution_methods):
print(attr['name'].ljust(20), output_deviation_mean[ind, :, 0])
print()
with np.printoptions(precision=3,
formatter={'float': '{: 0.3f}'.format},
suppress=True,
linewidth=np.inf):
print(
"Affect of removal of least important pixels at:- \npercentiles ",
percentiles)
for ind, attr in enumerate(attribution_methods):
print(attr['name'].ljust(20), output_deviation_mean[ind, :, 1])
def get_validation_dataloader(self, data_args) -> DataLoader:
self.mode = 'validation'
return torch.utils.data.DataLoader(deepcopy(self),
batch_size=data_args['batch_size'],
shuffle=data_args['shuffle'],
num_workers=get_cores_count())