def eval_all(base_model, dataloader, attack, exp: Experiment): adv = eval_adversarial_dataset(base_model, dataloader, attack, True) assert (base_model.pnml_model is False) pnml_model = exp.get_pnml_model(base_model, pnml_model_keep_grad=False) adv_pnml = eval_pnml_blackbox(pnml_model, adv, exp) natural = eval_adversarial_dataset( base_model, dataloader, get_attack({'attack_type': 'natural'}, base_model), False) natural_pnml = eval_adversarial_dataset( pnml_model, dataloader, get_attack({'attack_type': 'natural'}, pnml_model), False) return adv, adv_pnml, natural, natural_pnml
def eval_model(cls, model, dataloader, attack=get_attack({"attack_type": "no_attack"}), loss_func='default'): """ Evaluate the performance of the model on the train/test sets. :param model: the model that will be evaluated. :param dataloader: trainset or testset on which the evaluation will executed. :return: the loss and accuracy on the testset. """ model.eval() loss = 0 correct = 0 # with torch.no_grad(): for iter_num, (data, labels) in enumerate(dataloader): print("eval_model iter_num: {}".format(iter_num)) data, labels = TorchUtils.to_device(data), TorchUtils.to_device( labels) # with torch.enable_grad(): data = attack.create_adversarial_sample(data, labels) outputs, batch_loss = cls.__forward_pass(model, data, labels, loss_func) loss += float(batch_loss.detach_()) * len( data) # loss sum for all the batch _, predicted = torch.max(outputs.detach_().data, 1) correct += (predicted == labels).sum().item() if (predicted == labels).sum().item() == 1: print("correct prediction in iter_num: {}".format(iter_num)) acc = correct / len(dataloader.dataset) loss /= len(dataloader.dataset) return loss, acc
def get_adv_dataloaders(self, datafolder: str = './data', p=None, model=None): """ :param datafolder: location of the data :param p: (dict) the adversarial attack parameters :param model: the black/white-box model on which the attack will work, if None no attack will run :return: dataloaders dict """ if p is None: p = {'attack_type': "no_attack"} if model is None or p['attack_type'] == "no_attack": attack = get_attack(p) else: model.eval() attack = get_attack(p, model, get_dataset_min_max_val(self.exp_type)) return self._create_dataloaders(datafolder, attack)
def eval_pnml_blackbox(pnml_model, adv, exp: Experiment): dataloader = exp.get_blackbox_dataloader(adv) return eval_adversarial_dataset( pnml_model, dataloader['test'], get_attack({'attack_type': 'natural'}, pnml_model), False)
def train_model(self, model, dataloaders, num_epochs: int = 10, acc_goal=None, eval_test_every_n_epoch: int = 1, sample_test_data=None, sample_test_true_label=None): """ Train DNN model using some trainset. :param model: the model which will be trained. :param dataloaders: contains the trainset for training and testset for evaluation. :param num_epochs: number of epochs to train the model. :param acc_goal: stop training when getting to this accuracy rate on the trainset. :return: trained model (also the training of the models happen inplace) and the loss of the trainset and testset. """ self.logger.info("Use device:" + TorchUtils.get_device()) model = TorchUtils.to_device(model) attack = get_attack( self.attack_params, model, get_dataset_min_max_val(dataloaders['dataset_name'])) # If testset is already adversarial then do nothing else use the same attack to generate adversarial testset testset_attack = get_attack({ "attack_type": "no_attack" }) if dataloaders[ 'adv_test_flag'] else attack # TODO: replace training attack with testing attack epoch_time = 0 # Loop on epochs for epoch in range(1, num_epochs + 1): epoch_start_time = time.time() total_loss_in_epoch, natural_train_loss, train_acc = self.__train( model, dataloaders['train'], attack, sample_test_data, sample_test_true_label) # Evaluate testset if self.eval_test_during_train is True and epoch % eval_test_every_n_epoch == 0: test_loss, test_acc = self.eval_model(model, dataloaders['test'], testset_attack) else: test_loss, test_acc = torch.tensor([-1.]), torch.tensor([-1.]) epoch_time = time.time() - epoch_start_time # Save model if epoch % self.save_model_every_n_epoch == 0: torch.save( model.state_dict(), os.path.join(self.logger.output_folder, 'model_iter_%d.pt' % epoch)) # Log for param_group in self.optimizer.param_groups: lr = param_group['lr'] self.logger.info( '[%d/%d] [train test] loss =[%f %f] natural_train_loss=[%f], acc=[%f %f], lr=%f, epoch_time=%.2f' % (epoch, num_epochs, total_loss_in_epoch, test_loss, natural_train_loss, train_acc, test_acc, lr, epoch_time)) # Stop training if desired goal is achieved if acc_goal is not None and train_acc >= acc_goal: break test_loss, test_acc = self.eval_model(model, dataloaders['test'], testset_attack) train_loss_output = float(total_loss_in_epoch) test_loss_output = float(test_loss) # Print and save self.logger.info( '----- [train test] loss =[%f %f], natural_train_loss=[%f], acc=[%f %f] epoch_time=%.2f' % (total_loss_in_epoch, test_loss, natural_train_loss, train_acc, test_acc, epoch_time)) return model, train_loss_output, test_loss_output
def execute_pnml_adv_fix(pnml_params: dict, params_init_training: dict, dataloaders_input: dict, sample_test_data_trans, sample_test_true_label, idx: int, model_base_input, logger, genie_only_training: bool = False): """ Execute the PNML procedure: for each label train the model and save the prediction afterword. :param pnml_params: parameters of training the model for each label :param train_class: the train_class is used to train and eval the model :param dataloaders_input: dataloaders which contains the trainset :param sample_test_data_trans: the data of the test sample that will be evaluated :param sample_test_true_label: the true label of the test sample :param idx: the index in the testset dataset of the test sample :param model_base_input: the base model from which the train will start :param logger: logger class to print logs and save results to file :param genie_only_training: calculate only genie probability for speed up when debugging :return: None """ assert ( model_base_input.__class__ != "PnmlMnistClassifier" ) #deepcopy doesn't work for PnmlMnistClassifier but it doesn't matter because PnmlMnistClassifier accuracy is already pNML accuracy and this function doesn't need to run # Check pnml_params contains all required keys required_keys = [ 'lr', 'momentum', 'step_size', 'gamma', 'weight_decay', 'epochs' ] for key in required_keys: if key not in pnml_params: logger.logger.error('The key: %s is not in pnml_params' % key) raise ValueError('The key: %s is not in pnml_params' % key) classes_trained = dataloaders_input['classes'] if 'classes_cifar100' in dataloaders_input: classes_true = dataloaders_input['classes_cifar100'] elif 'classes_svhn' in dataloaders_input: classes_true = dataloaders_input['classes_svhn'] elif 'classes_noise' in dataloaders_input: classes_true = dataloaders_input['classes_noise'] else: classes_true = classes_trained # Iteration of all labels if genie_only_training: trained_label_list = [sample_test_true_label.tolist()] else: trained_label_list = range(len(classes_trained)) model = deepcopy(model_base_input) model.eval() refinement = get_attack(pnml_params['fix_type'], model, pnml_params['epsilon'], pnml_params['pgd_iter'], pnml_params['pgd_step'], get_dataset_min_max_val( dataloaders_input['dataset_name']), pnml_params['pgd_test_restart_num'], flip_grad_ratio=0.0) for fix_to_label in trained_label_list: time_trained_label_start = time.time() fix_label_expand = TorchUtils.to_device( torch.tensor(np.expand_dims(fix_to_label, 0), dtype=torch.int64)) true_label_expand = TorchUtils.to_device( torch.tensor(np.expand_dims(sample_test_true_label, 0), dtype=torch.int64) ) # make the label to tensor array type (important for loss calculation) if len(sample_test_data_trans.shape) == 3: sample_test_data_trans = TorchUtils.to_device( sample_test_data_trans.unsqueeze( 0)) # make the single sample 4-dim tensor time_trained_label = time.time() - time_trained_label_start # Evaluate with base model assert (not model.training) x_refine = refinement.create_adversarial_sample( sample_test_data_trans, true_label_expand, fix_label_expand) # assert(sample_test_data_trans.grad is None) prob, pred = eval_single_sample(model, x_refine) global execute_pnml_adv_fix_ind if execute_pnml_adv_fix_ind == 0: from utilities import plt_img # plt_img(x_refine, 0) # Save to file logger.add_entry_to_results_dict(idx, str(fix_to_label), prob, -1, -1) logger.info( 'idx=%d fix_to_label=[%d,%s], true_label=[%d,%s] predict=[%d], time=%4.2f[s]' % (idx, fix_to_label, classes_trained[fix_to_label], sample_test_true_label, classes_true[sample_test_true_label], np.argmax(prob), time_trained_label)) execute_pnml_adv_fix_ind = execute_pnml_adv_fix_ind + 1
def get_attack_for_model(self, model): return get_attack(self.params["adv_attack_test"], model, get_dataset_min_max_val(self.exp_type))