def eval_model(cls,
model,
dataloader,
attack=get_attack({"attack_type": "no_attack"}),
loss_func='default'):
"""
Evaluate the performance of the model on the train/test sets.
:param model: the model that will be evaluated.
:param dataloader: trainset or testset on which the evaluation will executed.
:return: the loss and accuracy on the testset.
"""
model.eval()
loss = 0
correct = 0
# with torch.no_grad():
for iter_num, (data, labels) in enumerate(dataloader):
print("eval_model iter_num: {}".format(iter_num))
data, labels = TorchUtils.to_device(data), TorchUtils.to_device(
labels)
# with torch.enable_grad():
data = attack.create_adversarial_sample(data, labels)
outputs, batch_loss = cls.__forward_pass(model, data, labels,
loss_func)
loss += float(batch_loss.detach_()) * len(
data) # loss sum for all the batch
_, predicted = torch.max(outputs.detach_().data, 1)
correct += (predicted == labels).sum().item()
if (predicted == labels).sum().item() == 1:
print("correct prediction in iter_num: {}".format(iter_num))
acc = correct / len(dataloader.dataset)
loss /= len(dataloader.dataset)
return loss, acc
def eval_batch(self, data, labels, enable_grad: bool = True):
"""
:param data: the data to evaluate
:param labels: the labels of the data
:param enable_grad: Should grad be enabled for later differentiation
:param model_output_type: "logits" if model output logits or "prob"
:param loss_func: The loss function used to calculate the loss
:return: batch loss, probability and label prediction.
"""
loss_func = torch.nn.NLLLoss(reduction='none')
self.eval()
with torch.set_grad_enabled(enable_grad):
data, labels = TorchUtils.to_device(data), TorchUtils.to_device(
labels)
output = self.__call__(data)
if not self.pnml_model: # Output is logits
prob = torch.softmax(output, 1)
loss = loss_func(
torch.log_softmax(output, 1),
labels) # log soft-max is more numerically stable
else: # Output is probability
prob = output
loss = loss_func(torch.log(output), labels)
# _, predicted_label = torch.max(output.data, 1)
return loss, prob, self.get_genie_prob()
def eval_adversarial_dataset(model,
dataloader,
attack,
save_adv_sample: bool = False):
"""
Evaluate model performance on dataloader with attack
:param model:
:param dataloader:
:param attack:
:param save_adv_sample:
:return:
"""
try:
torch.cuda.reset_max_memory_allocated()
except:
pass
logger = logger_utilities.get_logger()
model.eval(
) # TODO: model isn't required because model.eval() is done inside attack (make sure before removing)
adversarials = None
logger.info("Starting eval...")
if save_adv_sample:
logger.info("Save adversarial samples generated")
adv_batch_l = []
for iter_num, (data, labels) in enumerate(dataloader):
t0 = time.time()
data, labels = TorchUtils.to_device(data), TorchUtils.to_device(labels)
adversarials_batch = attack.create_adversarial_sample(
data,
labels,
get_adversarial_class=True,
save_adv_sample=save_adv_sample)
t1 = time.time()
adv_batch_l.append(adversarials_batch)
logger.info("eval_model iter_num: {} ,process time: {} ".format(
iter_num, t1 - t0))
# if iter_num == 0:
# adversarials = adversarials_batch
# else:
# adversarials.merge(adversarials_batch)
# t2 = time.time()
# logger.info("eval_model iter_num: {} ,process time: {} save time: {} ".format(iter_num, t1-t0, t2-t1))
adversarials = adv_batch_l[0].cat(adv_batch_l)
try:
logger.info(
"Max GPU memory allocated during eval_adversarial_dataset: {} MB".
format(torch.cuda.max_memory_allocated() / 2**20))
except:
pass
return adversarials
def __init__(self, mean: list, std: list):
"""
:param mean: a list representing the mean values in each channel
:param std: a list representing the std values in each channel
"""
super(NormalizeCls, self).__init__()
assert (type(mean) == list and type(std) == list
and len(mean) == len(std))
channels = len(mean)
self.mean = TorchUtils.to_device(
torch.tensor(mean, dtype=torch.float).reshape([channels, 1, 1]))
self.std = TorchUtils.to_device(
torch.tensor(std, dtype=torch.float).reshape([channels, 1, 1]))
def eval_single_sample(model, test_sample_data):
"""
Predict the probabilities assignment of the test sample by the model
:param model: the model which will evaluate the test sample
:param test_sample_data: the data of the test sample
:return: prob: probabilities vector. pred: the class prediction of the test sample
"""
# test_sample = (data, label)
# Test the sample
model.eval()
# if next(model.parameters()).is_cuda: # This only checks the first iteration from parameters(), a better way is to loop over all parameters
# sample_data = test_sample_data.cuda()
# else:
# sample_data = test_sample_data.cpu()
sample_data = TorchUtils.to_device(test_sample_data)
if len(sample_data.shape) == 3:
sample_data = sample_data.unsqueeze(
0) # make the single sample 4-dim tensor
# with torch.no_grad():
with torch.enable_grad():
output = model(sample_data).detach().cpu()
# Prediction
pred = output.max(1, keepdim=True)[1]
pred = pred.numpy()[0][0]
# Extract prob
prob = F.softmax(output, dim=-1)
prob = prob.numpy().tolist()[0]
return prob, pred
def calc_statistics(model, adv_l, logger):
"""
Calculate statistics from from adv_l
:param adv_l: a repacked adversarial list (see repack_adversarial_results func)
"""
### Calculate attack unsuccessful rate ###
# Evaluate
index = 0
batch_size = 32
correct = 0
while index < len(adv_l):
batch = adv_l[index:(index + batch_size)]
data_batch = torch.Tensor(unpack_adv_obj(batch))
data_batch = TorchUtils.to_device(data_batch)
labels_batch = np.stack([a['original_class'] for a in batch])
outputs = model(data_batch)
_, predicted = torch.max(outputs.detach_().data, 1)
# print(predicted)
# print(labels_batch)
correct += (predicted.cpu().numpy() == labels_batch).sum().item()
index += batch_size
unsuccessful_attack = correct / len(adv_l)
logger.info("Unsuccessful attacks {}%".format(100 * unsuccessful_attack))
# print(np.mean(model.forward(adv_img).argmax(axis=-1) == labels))
### - END - ###
### Calculate median distance ###
distances = np.asarray([a["distance"] for a in adv_l])
logger.info("Distance - min:{:.1e}, median:{:.1e}, max:{:.1e}".format(
distances.min(), np.median(distances), distances.max()))
logger.info("{} of {} attacks failed".format(
sum(a["distance"] == np.inf for a in adv_l), len(adv_l)))
logger.info("{} of {} inputs misclassified without perturbation".format(
sum(a["distance"] == 0 for a in adv_l), len(adv_l)))
# Present number of queries for the first 10 samples
for i in range(10):
print("Number of queries for {} sample: {}".format(
i, adv_l[i]['queries']))
def load_pretrained_imagenet_model(model_name: str, pretrained: bool = True):
"""
:param model_name: Could be one of the following:
'alexnet',
'densenet',
'densenet121',
'densenet161',
'densenet169',
'densenet201',
'inception',
'inception_v3',
'resnet',
'resnet101',
'resnet152',
'resnet18',
'resnet34',
'resnet50',
'squeezenet',
'squeezenet1_0',
'squeezenet1_1',
'vgg',
'vgg11',
'vgg11_bn',
'vgg13',
'vgg13_bn',
'vgg16',
'vgg16_bn',
'vgg19',
'vgg19_bn'
:param: pretrained: Whether to return a pretrained model or not.
:return: the trained model
"""
model = getattr(models, model_name)(
pretrained=pretrained
) # equivalent to models.resnet101(pretrained=True) for example
# model_with_norm = ImagenetModel(model)
return TorchUtils.to_device(model)
def __init__(self):
super(ModelTemplate, self).__init__()
self.pnml_model = False
self.regularization = TorchUtils.to_device(torch.zeros([1]))
self.ckpt_path = None
def train_model(self,
model,
dataloaders,
num_epochs: int = 10,
acc_goal=None,
eval_test_every_n_epoch: int = 1,
sample_test_data=None,
sample_test_true_label=None):
"""
Train DNN model using some trainset.
:param model: the model which will be trained.
:param dataloaders: contains the trainset for training and testset for evaluation.
:param num_epochs: number of epochs to train the model.
:param acc_goal: stop training when getting to this accuracy rate on the trainset.
:return: trained model (also the training of the models happen inplace)
and the loss of the trainset and testset.
"""
self.logger.info("Use device:" + TorchUtils.get_device())
model = TorchUtils.to_device(model)
attack = get_attack(
self.attack_params, model,
get_dataset_min_max_val(dataloaders['dataset_name']))
# If testset is already adversarial then do nothing else use the same attack to generate adversarial testset
testset_attack = get_attack({
"attack_type": "no_attack"
}) if dataloaders[
'adv_test_flag'] else attack # TODO: replace training attack with testing attack
epoch_time = 0
# Loop on epochs
for epoch in range(1, num_epochs + 1):
epoch_start_time = time.time()
total_loss_in_epoch, natural_train_loss, train_acc = self.__train(
model, dataloaders['train'], attack, sample_test_data,
sample_test_true_label)
# Evaluate testset
if self.eval_test_during_train is True and epoch % eval_test_every_n_epoch == 0:
test_loss, test_acc = self.eval_model(model,
dataloaders['test'],
testset_attack)
else:
test_loss, test_acc = torch.tensor([-1.]), torch.tensor([-1.])
epoch_time = time.time() - epoch_start_time
# Save model
if epoch % self.save_model_every_n_epoch == 0:
torch.save(
model.state_dict(),
os.path.join(self.logger.output_folder,
'model_iter_%d.pt' % epoch))
# Log
for param_group in self.optimizer.param_groups:
lr = param_group['lr']
self.logger.info(
'[%d/%d] [train test] loss =[%f %f] natural_train_loss=[%f], acc=[%f %f], lr=%f, epoch_time=%.2f'
% (epoch, num_epochs, total_loss_in_epoch, test_loss,
natural_train_loss, train_acc, test_acc, lr, epoch_time))
# Stop training if desired goal is achieved
if acc_goal is not None and train_acc >= acc_goal:
break
test_loss, test_acc = self.eval_model(model, dataloaders['test'],
testset_attack)
train_loss_output = float(total_loss_in_epoch)
test_loss_output = float(test_loss)
# Print and save
self.logger.info(
'----- [train test] loss =[%f %f], natural_train_loss=[%f], acc=[%f %f] epoch_time=%.2f'
% (total_loss_in_epoch, test_loss, natural_train_loss, train_acc,
test_acc, epoch_time))
return model, train_loss_output, test_loss_output
def execute_pnml_adv_fix(pnml_params: dict,
params_init_training: dict,
dataloaders_input: dict,
sample_test_data_trans,
sample_test_true_label,
idx: int,
model_base_input,
logger,
genie_only_training: bool = False):
"""
Execute the PNML procedure: for each label train the model and save the prediction afterword.
:param pnml_params: parameters of training the model for each label
:param train_class: the train_class is used to train and eval the model
:param dataloaders_input: dataloaders which contains the trainset
:param sample_test_data_trans: the data of the test sample that will be evaluated
:param sample_test_true_label: the true label of the test sample
:param idx: the index in the testset dataset of the test sample
:param model_base_input: the base model from which the train will start
:param logger: logger class to print logs and save results to file
:param genie_only_training: calculate only genie probability for speed up when debugging
:return: None
"""
assert (
model_base_input.__class__ != "PnmlMnistClassifier"
) #deepcopy doesn't work for PnmlMnistClassifier but it doesn't matter because PnmlMnistClassifier accuracy is already pNML accuracy and this function doesn't need to run
# Check pnml_params contains all required keys
required_keys = [
'lr', 'momentum', 'step_size', 'gamma', 'weight_decay', 'epochs'
]
for key in required_keys:
if key not in pnml_params:
logger.logger.error('The key: %s is not in pnml_params' % key)
raise ValueError('The key: %s is not in pnml_params' % key)
classes_trained = dataloaders_input['classes']
if 'classes_cifar100' in dataloaders_input:
classes_true = dataloaders_input['classes_cifar100']
elif 'classes_svhn' in dataloaders_input:
classes_true = dataloaders_input['classes_svhn']
elif 'classes_noise' in dataloaders_input:
classes_true = dataloaders_input['classes_noise']
else:
classes_true = classes_trained
# Iteration of all labels
if genie_only_training:
trained_label_list = [sample_test_true_label.tolist()]
else:
trained_label_list = range(len(classes_trained))
model = deepcopy(model_base_input)
model.eval()
refinement = get_attack(pnml_params['fix_type'],
model,
pnml_params['epsilon'],
pnml_params['pgd_iter'],
pnml_params['pgd_step'],
get_dataset_min_max_val(
dataloaders_input['dataset_name']),
pnml_params['pgd_test_restart_num'],
flip_grad_ratio=0.0)
for fix_to_label in trained_label_list:
time_trained_label_start = time.time()
fix_label_expand = TorchUtils.to_device(
torch.tensor(np.expand_dims(fix_to_label, 0), dtype=torch.int64))
true_label_expand = TorchUtils.to_device(
torch.tensor(np.expand_dims(sample_test_true_label, 0),
dtype=torch.int64)
) # make the label to tensor array type (important for loss calculation)
if len(sample_test_data_trans.shape) == 3:
sample_test_data_trans = TorchUtils.to_device(
sample_test_data_trans.unsqueeze(
0)) # make the single sample 4-dim tensor
time_trained_label = time.time() - time_trained_label_start
# Evaluate with base model
assert (not model.training)
x_refine = refinement.create_adversarial_sample(
sample_test_data_trans, true_label_expand, fix_label_expand)
# assert(sample_test_data_trans.grad is None)
prob, pred = eval_single_sample(model, x_refine)
global execute_pnml_adv_fix_ind
if execute_pnml_adv_fix_ind == 0:
from utilities import plt_img
# plt_img(x_refine, 0)
# Save to file
logger.add_entry_to_results_dict(idx, str(fix_to_label), prob, -1, -1)
logger.info(
'idx=%d fix_to_label=[%d,%s], true_label=[%d,%s] predict=[%d], time=%4.2f[s]'
% (idx, fix_to_label, classes_trained[fix_to_label],
sample_test_true_label, classes_true[sample_test_true_label],
np.argmax(prob), time_trained_label))
execute_pnml_adv_fix_ind = execute_pnml_adv_fix_ind + 1
def __train(self,
model,
train_loader,
attack,
sample_test_data=None,
sample_test_true_label=None):
"""
Execute one epoch of training
:param model: the model that will be trained.
:param train_loader: contains the trainset to train.
:return: the loss and accuracy of the trainset.
"""
model.train()
# Turn off batch normalization update
if self.freeze_batch_norm is True: # this works during fine-tuning because it can change the model even if LR=0.
model = model.apply(
set_bn_eval
) # this fucntion calls model.eval() which only effects dropout and batchnorm which will work in eval mode.
total_loss_in_epoch = 0
correct = 0
natural_loss = 0
natural_train_loss_in_ep = 0 if self.adv_learn_alpha != 1 else -1 * len(
train_loader.dataset)
natural_correct = 0
loss_sample_test = 0
# max_iter = np.ceil(len(train_loader.dataset) / train_loader.batch_size) #TODO: from some reason I am missing the last batch but the dataloader should not drop the last batch
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if sample_test_data is not None:
sample_test_data, sample_test_true_label = TorchUtils.to_device(
sample_test_data), TorchUtils.to_device(sample_test_true_label)
sample_test_data.requires_grad = False
sample_test_true_label.requires_grad = False
# Iterate over dataloaders
self.logger.init_debug_time_measure()
self.logger.debug("testing...")
for iter_num, (images, labels) in enumerate(train_loader):
self.logger.debug("iter: {}, load data".format(iter_num))
# Adjust to CUDA
images, labels = TorchUtils.to_device(
images), TorchUtils.to_device(labels)
self.logger.debug(
"iter: {}, data and labels to CUDA:".format(iter_num))
# Forward-pass of natural images
self.optimizer.zero_grad()
if self.adv_learn_alpha != 1:
outputs, natural_loss = self.__forward_pass(
model, images, labels)
_, predicted = torch.max(outputs.data, 1)
natural_correct += (predicted == labels).sum().item()
natural_train_loss_in_ep += natural_loss * len(
images) # natural_loss sum for the epoch
self.logger.debug("iter: {}, Forward pass".format(iter_num))
# Back-propagation
if self.adv_learn_alpha == 0:
natural_loss.backward()
self.logger.debug("iter: {}, Backward pass".format(iter_num))
correct = natural_correct
else:
# # The loss is averaged over the minibatch, this doesn't matter at all since the loss magnitude
# # is only just to determine gradient sign. Each pixel is changed by -+epsilon, no matter
# # it's gradient magnitude.
adv_images = attack.create_adversarial_sample(images, labels)
self.optimizer.zero_grad()
self.logger.debug(
"iter: {}, create adversarial data".format(iter_num))
#### add another sample
# TODO: move it outside if, otherwise this won't happen for non-adversarial training
if (iter_num) == 0 and sample_test_data is not None:
sample_test_data.requires_grad = False
sample_test_true_label.requires_grad = False
output_sample_test, loss_sample_test = self.__forward_pass(
model, sample_test_data, sample_test_true_label)
_, predicted = torch.max(output_sample_test.data, 1)
# correct += (predicted == sample_test_true_label).sum().item() #TODO: when calculating accuracy (after epoch) we need to divide by +1
if TrainClass.criterion.reduction == 'elementwise_mean': # Re-average the loss since another image was added
loss_sample_test = loss_sample_test / (len(images) + 1)
# natural_loss = natural_loss * len(images) / (len(images) + 1) TODO: uncomment
loss_sample_test.backward(retain_graph=True)
# self.optimizer.zero_grad()
####
outputs, adv_loss = self.__forward_pass(
model, adv_images, labels)
self.logger.debug(
"iter: {}, Adv. Forward pass".format(iter_num))
_, predicted = torch.max(outputs.data, 1)
correct += (predicted == labels).sum().item()
total_loss = (
1 - self.adv_learn_alpha
) * natural_loss + self.adv_learn_alpha * adv_loss #+ loss_sample_test
total_loss_in_epoch += total_loss * len(images)
total_loss.backward()
self.logger.debug(
"iter: {}, Adv. Backward pass".format(iter_num))
self.optimizer.step()
self.logger.debug("iter: {}, Optimizer step".format(iter_num))
self.scheduler.step()
natural_train_loss_in_ep /= len(train_loader.dataset)
total_loss_in_epoch /= len(train_loader.dataset)
train_acc = correct / len(train_loader.dataset)
return total_loss_in_epoch, natural_train_loss_in_ep, train_acc