def eval_all(base_model, dataloader, attack, exp: Experiment):
adv = eval_adversarial_dataset(base_model, dataloader, attack, True)
assert (base_model.pnml_model is False)
pnml_model = exp.get_pnml_model(base_model, pnml_model_keep_grad=False)
adv_pnml = eval_pnml_blackbox(pnml_model, adv, exp)
natural = eval_adversarial_dataset(
base_model, dataloader,
get_attack({'attack_type': 'natural'}, base_model), False)
natural_pnml = eval_adversarial_dataset(
pnml_model, dataloader,
get_attack({'attack_type': 'natural'}, pnml_model), False)
return adv, adv_pnml, natural, natural_pnml
def eval_model(cls,
model,
dataloader,
attack=get_attack({"attack_type": "no_attack"}),
loss_func='default'):
"""
Evaluate the performance of the model on the train/test sets.
:param model: the model that will be evaluated.
:param dataloader: trainset or testset on which the evaluation will executed.
:return: the loss and accuracy on the testset.
"""
model.eval()
loss = 0
correct = 0
# with torch.no_grad():
for iter_num, (data, labels) in enumerate(dataloader):
print("eval_model iter_num: {}".format(iter_num))
data, labels = TorchUtils.to_device(data), TorchUtils.to_device(
labels)
# with torch.enable_grad():
data = attack.create_adversarial_sample(data, labels)
outputs, batch_loss = cls.__forward_pass(model, data, labels,
loss_func)
loss += float(batch_loss.detach_()) * len(
data) # loss sum for all the batch
_, predicted = torch.max(outputs.detach_().data, 1)
correct += (predicted == labels).sum().item()
if (predicted == labels).sum().item() == 1:
print("correct prediction in iter_num: {}".format(iter_num))
acc = correct / len(dataloader.dataset)
loss /= len(dataloader.dataset)
return loss, acc
def get_adv_dataloaders(self,
datafolder: str = './data',
p=None,
model=None):
"""
:param datafolder: location of the data
:param p: (dict) the adversarial attack parameters
:param model: the black/white-box model on which the attack will work, if None no attack will run
:return: dataloaders dict
"""
if p is None:
p = {'attack_type': "no_attack"}
if model is None or p['attack_type'] == "no_attack":
attack = get_attack(p)
else:
model.eval()
attack = get_attack(p, model,
get_dataset_min_max_val(self.exp_type))
return self._create_dataloaders(datafolder, attack)
def eval_pnml_blackbox(pnml_model, adv, exp: Experiment):
dataloader = exp.get_blackbox_dataloader(adv)
return eval_adversarial_dataset(
pnml_model, dataloader['test'],
get_attack({'attack_type': 'natural'}, pnml_model), False)
def train_model(self,
model,
dataloaders,
num_epochs: int = 10,
acc_goal=None,
eval_test_every_n_epoch: int = 1,
sample_test_data=None,
sample_test_true_label=None):
"""
Train DNN model using some trainset.
:param model: the model which will be trained.
:param dataloaders: contains the trainset for training and testset for evaluation.
:param num_epochs: number of epochs to train the model.
:param acc_goal: stop training when getting to this accuracy rate on the trainset.
:return: trained model (also the training of the models happen inplace)
and the loss of the trainset and testset.
"""
self.logger.info("Use device:" + TorchUtils.get_device())
model = TorchUtils.to_device(model)
attack = get_attack(
self.attack_params, model,
get_dataset_min_max_val(dataloaders['dataset_name']))
# If testset is already adversarial then do nothing else use the same attack to generate adversarial testset
testset_attack = get_attack({
"attack_type": "no_attack"
}) if dataloaders[
'adv_test_flag'] else attack # TODO: replace training attack with testing attack
epoch_time = 0
# Loop on epochs
for epoch in range(1, num_epochs + 1):
epoch_start_time = time.time()
total_loss_in_epoch, natural_train_loss, train_acc = self.__train(
model, dataloaders['train'], attack, sample_test_data,
sample_test_true_label)
# Evaluate testset
if self.eval_test_during_train is True and epoch % eval_test_every_n_epoch == 0:
test_loss, test_acc = self.eval_model(model,
dataloaders['test'],
testset_attack)
else:
test_loss, test_acc = torch.tensor([-1.]), torch.tensor([-1.])
epoch_time = time.time() - epoch_start_time
# Save model
if epoch % self.save_model_every_n_epoch == 0:
torch.save(
model.state_dict(),
os.path.join(self.logger.output_folder,
'model_iter_%d.pt' % epoch))
# Log
for param_group in self.optimizer.param_groups:
lr = param_group['lr']
self.logger.info(
'[%d/%d] [train test] loss =[%f %f] natural_train_loss=[%f], acc=[%f %f], lr=%f, epoch_time=%.2f'
% (epoch, num_epochs, total_loss_in_epoch, test_loss,
natural_train_loss, train_acc, test_acc, lr, epoch_time))
# Stop training if desired goal is achieved
if acc_goal is not None and train_acc >= acc_goal:
break
test_loss, test_acc = self.eval_model(model, dataloaders['test'],
testset_attack)
train_loss_output = float(total_loss_in_epoch)
test_loss_output = float(test_loss)
# Print and save
self.logger.info(
'----- [train test] loss =[%f %f], natural_train_loss=[%f], acc=[%f %f] epoch_time=%.2f'
% (total_loss_in_epoch, test_loss, natural_train_loss, train_acc,
test_acc, epoch_time))
return model, train_loss_output, test_loss_output
def execute_pnml_adv_fix(pnml_params: dict,
params_init_training: dict,
dataloaders_input: dict,
sample_test_data_trans,
sample_test_true_label,
idx: int,
model_base_input,
logger,
genie_only_training: bool = False):
"""
Execute the PNML procedure: for each label train the model and save the prediction afterword.
:param pnml_params: parameters of training the model for each label
:param train_class: the train_class is used to train and eval the model
:param dataloaders_input: dataloaders which contains the trainset
:param sample_test_data_trans: the data of the test sample that will be evaluated
:param sample_test_true_label: the true label of the test sample
:param idx: the index in the testset dataset of the test sample
:param model_base_input: the base model from which the train will start
:param logger: logger class to print logs and save results to file
:param genie_only_training: calculate only genie probability for speed up when debugging
:return: None
"""
assert (
model_base_input.__class__ != "PnmlMnistClassifier"
) #deepcopy doesn't work for PnmlMnistClassifier but it doesn't matter because PnmlMnistClassifier accuracy is already pNML accuracy and this function doesn't need to run
# Check pnml_params contains all required keys
required_keys = [
'lr', 'momentum', 'step_size', 'gamma', 'weight_decay', 'epochs'
]
for key in required_keys:
if key not in pnml_params:
logger.logger.error('The key: %s is not in pnml_params' % key)
raise ValueError('The key: %s is not in pnml_params' % key)
classes_trained = dataloaders_input['classes']
if 'classes_cifar100' in dataloaders_input:
classes_true = dataloaders_input['classes_cifar100']
elif 'classes_svhn' in dataloaders_input:
classes_true = dataloaders_input['classes_svhn']
elif 'classes_noise' in dataloaders_input:
classes_true = dataloaders_input['classes_noise']
else:
classes_true = classes_trained
# Iteration of all labels
if genie_only_training:
trained_label_list = [sample_test_true_label.tolist()]
else:
trained_label_list = range(len(classes_trained))
model = deepcopy(model_base_input)
model.eval()
refinement = get_attack(pnml_params['fix_type'],
model,
pnml_params['epsilon'],
pnml_params['pgd_iter'],
pnml_params['pgd_step'],
get_dataset_min_max_val(
dataloaders_input['dataset_name']),
pnml_params['pgd_test_restart_num'],
flip_grad_ratio=0.0)
for fix_to_label in trained_label_list:
time_trained_label_start = time.time()
fix_label_expand = TorchUtils.to_device(
torch.tensor(np.expand_dims(fix_to_label, 0), dtype=torch.int64))
true_label_expand = TorchUtils.to_device(
torch.tensor(np.expand_dims(sample_test_true_label, 0),
dtype=torch.int64)
) # make the label to tensor array type (important for loss calculation)
if len(sample_test_data_trans.shape) == 3:
sample_test_data_trans = TorchUtils.to_device(
sample_test_data_trans.unsqueeze(
0)) # make the single sample 4-dim tensor
time_trained_label = time.time() - time_trained_label_start
# Evaluate with base model
assert (not model.training)
x_refine = refinement.create_adversarial_sample(
sample_test_data_trans, true_label_expand, fix_label_expand)
# assert(sample_test_data_trans.grad is None)
prob, pred = eval_single_sample(model, x_refine)
global execute_pnml_adv_fix_ind
if execute_pnml_adv_fix_ind == 0:
from utilities import plt_img
# plt_img(x_refine, 0)
# Save to file
logger.add_entry_to_results_dict(idx, str(fix_to_label), prob, -1, -1)
logger.info(
'idx=%d fix_to_label=[%d,%s], true_label=[%d,%s] predict=[%d], time=%4.2f[s]'
% (idx, fix_to_label, classes_trained[fix_to_label],
sample_test_true_label, classes_true[sample_test_true_label],
np.argmax(prob), time_trained_label))
execute_pnml_adv_fix_ind = execute_pnml_adv_fix_ind + 1
def get_attack_for_model(self, model):
return get_attack(self.params["adv_attack_test"], model,
get_dataset_min_max_val(self.exp_type))