def test(self, model, loader, adv_test=False): # adv_test is False, return adv_acc as -1 total_acc = 0.0 num = 0 total_adv_acc = 0.0 with torch.no_grad(): for data, label in loader: data, label = tensor2cuda(data), tensor2cuda(label) output = model(data, _eval=True) pred = torch.max(output, dim=1)[1] te_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy(), 'sum') total_acc += te_acc num += output.shape[0] if adv_test: # use predicted label as target label with torch.enable_grad(): adv_data = self.attack.perturb(data, pred, 'mean', False) adv_output = model(adv_data, _eval=True) adv_pred = torch.max(adv_output, dim=1)[1] adv_acc = evaluate(adv_pred.cpu().numpy(), label.cpu().numpy(), 'sum') total_adv_acc += adv_acc else: total_adv_acc = -num return total_acc / num , total_adv_acc / num
def perturb(self, original_images, labels, reduction4loss='mean', random_start=False): # original_images: values are within self.min_val and self.max_val # The adversaries created from random close points to the original data if random_start: rand_perturb = torch.FloatTensor(original_images.shape).uniform_( -self.epsilon, self.epsilon) rand_perturb = tensor2cuda(rand_perturb) x = original_images + rand_perturb x.clamp_(self.min_val, self.max_val) else: x = original_images.clone() x.requires_grad = True # max_x = original_images + self.epsilon # min_x = original_images - self.epsilon self.model.eval() with torch.enable_grad(): for _iter in range(self.max_iters): outputs = self.model(x, _eval=True) loss = F.cross_entropy(outputs, labels, reduction=reduction4loss) if reduction4loss == 'none': grad_outputs = tensor2cuda(torch.ones(loss.shape)) else: grad_outputs = None grads = torch.autograd.grad(loss, x, grad_outputs=grad_outputs, only_inputs=True)[0] x.data += self.alpha * torch.sign(grads.data) # the adversaries' pixel value should within max_x and min_x due # to the l_infinity / l2 restriction x = project(x, original_images, self.epsilon, self._type) # the adversaries' value should be valid pixel value x.clamp_(self.min_val, self.max_val) self.model.train() return x
def generate_gradients(self, input_image, target_class): # Put model in evaluation mode self.model.eval() x = input_image.clone() x.requires_grad = True with torch.enable_grad(): # Forward model_output = self.model(x) # Zero grads self.model.zero_grad() grad_outputs = one_hot(target_class, model_output.shape[1]) grad_outputs = tensor2cuda(grad_outputs) grad = torch.autograd.grad(model_output, x, grad_outputs=grad_outputs, only_inputs=True)[0] self.model.train() return grad
def cw_attack_test(model, te_loader): model.eval() cw_acc = 0.0 te_acc = 0.0 test_num = 0 adv_num = 0 total_te_acc = 0.0 total_cw_acc = 0.0 mean = [0] std = [1] inputs_box = (min((0 - m) / s for m, s in zip(mean, std)), max((1 - m) / s for m, s in zip(mean, std))) for data, label in te_loader: te_acc = 0.0 data, label = tensor2cuda(data), tensor2cuda(label) test_output = model(data, _eval=True) test_pred = torch.max(test_output, dim=1)[1] te_acc = evaluate(test_pred.cpu().numpy(), label.cpu().numpy(), 'sum') total_te_acc += te_acc test_num += test_output.shape[0] ####################################################################### # Adversarial Samples cw_acc = 0.0 attack = carlini_wagner_L2.L2Adversary(targeted=False, confidence=0.0, search_steps=10, box=inputs_box, optimizer_lr=5e-4) adv_data = attack(model, data, label, to_numpy=False) adv_output = model(adv_data.cuda(), _eval=True) adv_num += adv_output.shape[0] adv_pred = torch.max(adv_output, dim=1)[1] cw_acc = evaluate(adv_pred.cpu().numpy(), label.cpu().numpy(), 'sum') total_cw_acc += cw_acc print("Total Test Samples", test_num) print("Total Adversarial Samples", adv_num) print("Accuracy before CW Attack", ((total_te_acc / test_num) * 100)) print("Accuracy after CW Attack", ((total_cw_acc / adv_num) * 100))
def train(self, model, tr_loader, va_loader=None, adv_train=False): args = self.args logger = self.logger opt = torch.optim.Adam(model.parameters(), args.learning_rate, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[100, 150], gamma=0.1) _iter = 0 begin_time = time() for epoch in range(1, args.max_epoch+1): scheduler.step() for data, label in tr_loader: data, label = tensor2cuda(data), tensor2cuda(label) if adv_train: # When training, the adversarial example is created from a random # close point to the original data point. If in evaluation mode, # just start from the original data point. adv_data = self.attack.perturb(data, label, 'mean', True) output = model(adv_data, _eval=False) else: output = model(data, _eval=False) loss = F.cross_entropy(output, label) opt.zero_grad() loss.backward() opt.step() if _iter % args.n_eval_step == 0: t1 = time() if adv_train: with torch.no_grad(): stand_output = model(data, _eval=True) pred = torch.max(stand_output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 else: adv_data = self.attack.perturb(data, label, 'mean', False) with torch.no_grad(): adv_output = model(adv_data, _eval=True) pred = torch.max(adv_output, dim=1)[1] # print(label) # print(pred) adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 t2 = time() print('%.3f' % (t2 - t1)) logger.info('epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' % ( epoch, _iter, time()-begin_time, loss.item())) logger.info('standard acc: %.3f %%, robustness acc: %.3f %%' % ( std_acc, adv_acc)) # begin_time = time() # if va_loader is not None: # va_acc, va_adv_acc = self.test(model, va_loader, True) # va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0 # logger.info('\n' + '='*30 + ' evaluation ' + '='*30) # logger.info('test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % ( # va_acc, va_adv_acc, time() - begin_time)) # logger.info('='*28 + ' end of evaluation ' + '='*28 + '\n') begin_time = time() if _iter % args.n_store_image_step == 0: tv.utils.save_image(torch.cat([data.cpu(), adv_data.cpu()], dim=0), os.path.join(args.log_folder, 'images_%d.jpg' % _iter), nrow=16) if _iter % args.n_checkpoint_step == 0: file_name = os.path.join(args.model_folder, 'checkpoint_%d.pth' % _iter) save_model(model, file_name) _iter += 1 if va_loader is not None: t1 = time() va_acc, va_adv_acc = self.test(model, va_loader, True) va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0 t2 = time() logger.info('\n'+'='*20 +' evaluation at epoch: %d iteration: %d '%(epoch, _iter) \ +'='*20) logger.info('test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % ( va_acc, va_adv_acc, t2-t1)) logger.info('='*28+' end of evaluation '+'='*28+'\n')
def train(self, model, tr_loader, va_loader=None, adv_train=False): args = self.args logger = self.logger best_adv_acc = 0. opt = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=0.9, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[75, 90], gamma=0.1) _iter = 0 for epoch in range(1, args.max_epoch + 1): begin_time = time() scheduler.step() for data, label in tr_loader: data, label = tensor2cuda(data), tensor2cuda(label) if adv_train: # When training, the adversarial example is created from a random # close point to the original data point. If in evaluation mode, # just start from the original data point. adv_data = self.attack.perturb(data, label, 'mean', True, args.beta) output, class_wise_output = model(adv_data, y=label, _eval=False) loss = F.cross_entropy(output, label) channel_reg_loss = 0. for extra_output in class_wise_output: channel_reg_loss += F.cross_entropy( extra_output, label) channel_reg_loss = channel_reg_loss / len( class_wise_output) loss = loss + args.beta * channel_reg_loss else: output, class_wise_output = model(data, y=label, _eval=False) loss = F.cross_entropy(output, label) channel_reg_loss = 0. for extra_output in class_wise_output: channel_reg_loss += F.cross_entropy( extra_output, label) channel_reg_loss = channel_reg_loss / len( class_wise_output) loss = loss + args.beta * channel_reg_loss opt.zero_grad() loss.backward() opt.step() if _iter % args.n_eval_step == 0: if adv_train: with torch.no_grad(): stand_output, _ = model(data, _eval=True) pred = torch.max(stand_output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 else: adv_data = self.attack.perturb(data, label, 'mean', False) with torch.no_grad(): adv_output, _ = model(adv_data, _eval=True) pred = torch.max(adv_output, dim=1)[1] adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 logger.info( 'epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' % (epoch, _iter, time() - begin_time, loss.item())) logger.info( 'standard acc: %.3f %%, robustness acc: %.3f %%' % (std_acc, adv_acc)) if _iter % args.n_checkpoint_step == 0: file_name = os.path.join(args.model_folder, 'checkpoint_last.pth') save_model(model, file_name) _iter += 1 epoch_cost = time() print('cost:', epoch_cost - begin_time) # assert False if va_loader is not None: t1 = time() va_acc, va_adv_acc = self.test(model, va_loader, True, False) va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0 t2 = time() logger.info('\n' + '=' * 20 + ' evaluation at epoch: %d iteration: %d ' % (epoch, _iter) \ + '=' * 20) logger.info( 'test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % (va_acc, va_adv_acc, t2 - t1)) logger.info('=' * 28 + ' end of evaluation ' + '=' * 28 + '\n') if va_adv_acc > best_adv_acc: file_name = os.path.join(args.model_folder, 'checkpoint_best_adv.pth') save_model(model, file_name) best_adv_acc = va_adv_acc
def train(self, model, tr_loader, va_loader=None, adv_train=False): args = self.args logger = self.logger opt = torch.optim.Adam(model.parameters(), args.learning_rate, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[100, 200], gamma=0.1) _iter = 0 begin_time = time() best_loss = 999 for epoch in range(1, args.max_epoch+1): for data, label in tr_loader: # for sample in tr_loader: # data, label = sample['buffers'], sample['labels'] # print('data shape is ', data.shape) # print('label shape is ', label.shape) data, label = tensor2cuda(data), tensor2cuda(label) if adv_train: # When training, the adversarial example is created from a random # close point to the original data point. If in evaluation mode, # just start from the original data point. adv_data = self.attack.perturb(data, label, 'mean', True) model.train() output = model(adv_data) else: model.train() output = model(data) loss = F.binary_cross_entropy(torch.sigmoid(output), label) opt.zero_grad() loss.backward() opt.step() t = Variable(torch.Tensor([0.5]).cuda()) # threshold to compute accuracy if _iter % args.n_eval_step == 0: t1 = time() if adv_train: with torch.no_grad(): model.eval() stand_output = model(data) # pred = torch.max(stand_output, dim=1)[1] # this give us the indices tensor pred = torch.sigmoid(stand_output) out = (pred > t).float() # print(pred.shape) # print(out.shape) # std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 stdacc_list = evaluate_(out.cpu().numpy(), label.cpu().numpy()) # print('std accuracy list shape: ', np.array(stdacc_list).shape) # pred = torch.max(output, dim=1)[1] pred = torch.sigmoid(output) out = (pred > t).float() # print(pred) # adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 advacc_list = evaluate_(out.cpu().numpy(), label.cpu().numpy()) # print('adv accuracy list shape: ', np.array(stdacc_list).shape) else: adv_data = self.attack.perturb(data, label, 'mean', False) with torch.no_grad(): model.eval() adv_output = model(adv_data) # adv_output = model(adv_data, _eval=True) # pred = torch.max(adv_output, dim=1)[1] pred = torch.sigmoid(adv_output) out = (pred > t).float() # print(label) # print(pred) # adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 advacc_list = evaluate_(out.cpu().numpy(), label.cpu().numpy()) # pred = torch.max(output, dim=1)[1] pred = torch.sigmoid(output) out = (pred > t).float() # print(pred) # std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 stdacc_list = evaluate_(out.cpu().numpy(), label.cpu().numpy()) t2 = time() print('%.3f' % (t2 - t1)) logger.info('epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' % ( epoch, _iter, time()-begin_time, loss.item())) begin_time = time() if _iter % args.n_checkpoint_step == 0: file_name = os.path.join(args.model_folder, 'checkpoint_%d.pth' % _iter) save_model(model, file_name) _iter += 1 scheduler.step() if va_loader is not None: t1 = time() va_acc, va_adv_acc, va_stdloss, va_advloss = self.test(model, va_loader, True, False) va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0 if va_stdloss < best_loss: best_loss = va_stdloss file_name = os.path.join(args.model_folder, 'checkpoint_best.pth') save_model(model, file_name) t2 = time() logger.info('\n'+'='*20 +' evaluation at epoch: %d iteration: %d '%(epoch, _iter) \ +'='*20) logger.info('test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % ( va_acc, va_adv_acc, t2-t1)) logger.info('test loss: %.3f , test adv loss: %.3f , spent: %.3f' % ( va_stdloss, va_advloss, t2-t1)) logger.info('='*28+' end of evaluation '+'='*28+'\n')
def test(self, model, loader, adv_test=False, use_pseudo_label=False, if_AUC=False): # adv_test is False, return adv_acc as -1 total_acc = 0.0 num = 0 total_adv_acc = 0.0 total_stdloss = 0.0 total_advloss = 0.0 t = Variable(torch.Tensor([0.5]).cuda()) # threshold to compute accuracy label_list = [] pred_list = [] predadv_list = [] with torch.no_grad(): for data, label in loader: # for sample in loader: # data, label = sample['buffers'], sample['labels'] data, label = tensor2cuda(data), tensor2cuda(label) model.eval() output = model(data) std_loss = F.binary_cross_entropy(torch.sigmoid(output), label) pred = torch.sigmoid(output) out = (pred > t).float() te_acc = np.mean(evaluate_(out.cpu().numpy(), label.cpu().numpy())) total_acc += te_acc total_stdloss += std_loss if if_AUC: label_list.append(label.cpu().numpy()) pred_list.append(pred.cpu().numpy()) # num += output.shape[0] num += 1 if adv_test: # use predicted label as target label with torch.enable_grad(): adv_data = self.attack.perturb(data, pred if use_pseudo_label else label, 'mean', False) model.eval() adv_output = model(adv_data) adv_loss = F.binary_cross_entropy(torch.sigmoid(adv_output), label) adv_pred = torch.sigmoid(adv_output) if if_AUC: predadv_list.append(adv_pred.cpu().numpy()) adv_out = (adv_pred > t).float() adv_acc = np.mean(evaluate_(adv_out.cpu().numpy(), label.cpu().numpy())) total_adv_acc += adv_acc total_advloss += adv_loss else: total_adv_acc = -num if if_AUC: pred = np.squeeze(np.array(pred_list)) predadv = np.squeeze(np.array(predadv_list)) label = np.squeeze(np.array(label_list)) np.save(os.path.join(self.args.log_folder, 'y_pred.npy'), pred) # np.save(os.path.join(self.args.log_folder, 'y_predadv_'+str(args.epsilon)+'.npy'), predadv) np.save(os.path.join(self.args.log_folder, 'y_true.npy'), label) # PRED_label = ['No Finding', 'Cardiomegaly', 'Edema', # 'Consolidation', 'Pneumonia', 'Atelectasis', # 'Pneumothorax', 'Pleural Effusion'] # PRED_label = ['healthy', 'partially injured', 'completely ruptured'] # PRED_label = ['malignancy'] # plot_AUC(pred, label, self.args.log_folder, 'auc.png', PRED_label) # plot_AUC(predadv, label, self.args.log_folder, 'auc_'+str(args.epsilon)+'.png', PRED_label) # np.save('predstd_'+str(args.epsilon)+'.npy', pred) # np.save('predstdadv_'+str(args.epsilon)+'.npy', predadv) # np.save('labelstd_'+str(args.epsilon)+'.npy', label) else: return total_acc / num, total_adv_acc / num, total_stdloss / num, total_advloss / num
label_dict = LabelDict(args.dataset) te_dataset = tv.datasets.CIFAR10(args.data_root, train=False, transform=tv.transforms.ToTensor(), download=True) te_loader = DataLoader(te_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4) for data, label in te_loader: data, label = tensor2cuda(data), tensor2cuda(label) break adv_list = [] pred_list = [] with torch.no_grad(): model = WideResNet(depth=34, num_classes=10, widen_factor=10, dropRate=0.0) load_model(model, args.load_checkpoint) if torch.cuda.is_available(): model.cuda()
def test(self, model, loader, epsilon, adv_test=False, use_pseudo_label=False, if_AUC=False): # adv_test is False, return adv_acc as -1 total_acc = 0.0 num = 0 total_adv_acc = 0.0 total_stdloss = 0.0 total_advloss = 0.0 t = Variable(torch.Tensor([0.5 ]).cuda()) # threshold to compute accuracy label_list = [] pred_list = [] predadv_list = [] with torch.no_grad(): for data, label in loader: data, label = tensor2cuda(data), tensor2cuda(label) model.eval() output = model(data, [0]) std_loss = F.binary_cross_entropy(torch.sigmoid(output), label) pred = torch.sigmoid(output) out = (pred > t).float() te_acc = np.mean( evaluate_(out.cpu().numpy(), label.cpu().numpy())) total_acc += te_acc total_stdloss += std_loss if if_AUC: label_list.append(label.cpu().numpy()) pred_list.append(pred.cpu().numpy()) # num += output.shape[0] num += 1 if adv_test: # use predicted label as target label with torch.enable_grad(): adv_data = self.attack.perturb( data, pred if use_pseudo_label else label, 'mean', False, True) model.eval() adv_output = model(adv_data, [1]) adv_loss = F.binary_cross_entropy( torch.sigmoid(adv_output), label) adv_pred = torch.sigmoid(adv_output) if if_AUC: predadv_list.append(adv_pred.cpu().numpy()) adv_out = (adv_pred > t).float() adv_acc = np.mean( evaluate_(adv_out.cpu().numpy(), label.cpu().numpy())) total_adv_acc += adv_acc total_advloss += adv_loss else: total_adv_acc = -num if if_AUC: pred = np.squeeze(np.array(pred_list)) predadv = np.squeeze(np.array(predadv_list)) label = np.squeeze(np.array(label_list)) np.save(os.path.join(self.log_folder, 'y_pred.npy'), pred) np.save(os.path.join(self.log_folder, 'y_true.npy'), label) np.save( os.path.join(self.log_folder, 'y_predadv_' + str(epsilon) + '.npy'), predadv) else: return total_acc / num, total_adv_acc / num, total_stdloss / num, total_advloss / num
def train(self, model, tr_loader, va_loader=None, adv_train=False): args = self.args logger = self.logger opt = torch.optim.Adam(model.parameters(), args.learning_rate, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.MultiStepLR( opt, milestones=[50, 100, 150], gamma=0.1) _iter = 0 begin_time = time() # Loss mixer: (ce_loss_orig + (lambda1 * ce_loss_adversarial))/(1 + lambda1) + (lambda2 * custom_loss) lambda1 = 0.6 if args.custom_regularizer == 'Triplet': # 1 for triplet lambda2 = 1 elif args.custom_regularizer == 'Npairs': # 0.5 for npairs for now lambda2 = 0.5 else: lambda2 = 0 for epoch in range(1, args.max_epoch + 1): #scheduler.step() for data, label in tr_loader: data, label = tensor2cuda(data), tensor2cuda(label) if adv_train: # Outputs from original samples output = model(data, _eval=False) # When training, the adversarial example is created from a random # close point to the original data point. If in evaluation mode, # just start from the original data point. if args.adv_train_mode == 'FGSM': adv_data = self.train_attack.perturb( data, label, 'mean', True) elif args.adv_train_mode == 'CW': adv_data = self.train_attack(model, data, label, to_numpy=False) adv_data = adv_data.cuda() # Outputs from adversarial samples adv_output = model(adv_data, _eval=False) # Shape of pred: batch_size # Predicted labels on adversarial samples pred = torch.max(adv_output, dim=1)[1] if args.custom_regularizer == 'No_reg': tloss = 0 else: # Shape of output: batch_size * (64*widen_factor) X_output = model(data, _eval=False, _prefinal=True) X_adv_output = model(adv_data, _eval=False, _prefinal=True) if args.custom_regularizer == 'Triplet': tloss = triplet_loss(X_output, X_adv_output, label, pred, logger) elif args.custom_regularizer == 'Npairs': tloss = npairs_loss(X_output, X_adv_output, label) else: output = model(data, _eval=False) tloss = 0 ce_loss = F.cross_entropy(output, label) if adv_train: # Adding loss due to adversarial samples in the mix ce_loss = ce_loss + lambda1 * F.cross_entropy( adv_output, label) ce_loss = ce_loss / (1 + lambda1) loss = ce_loss + lambda2 * tloss opt.zero_grad() loss.backward() opt.step() if _iter % args.n_eval_step == 0: t1 = time() logger.info( 'Total Loss logger: %.3f, CE Loss: %.3f, %s loss: %.3f' % (loss, ce_loss, args.custom_regularizer, tloss)) if adv_train: with torch.no_grad(): stand_output = model(data, _eval=True) pred_stand = torch.max(stand_output, dim=1)[1] std_acc = evaluate(pred_stand.cpu().numpy(), label.cpu().numpy()) * 100 adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 else: if args.adv_train_mode == 'FGSM': adv_data = self.train_attack.perturb( data, label, 'mean', False) elif args.adv_train_mode == 'CW': adv_data = self.train_attack(model, data, label, to_numpy=False) adv_data = adv_data.cuda() with torch.no_grad(): adv_output = model(adv_data, _eval=True) pred = torch.max(adv_output, dim=1)[1] adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 t2 = time() print('%.3f' % (t2 - t1)) logger.info( 'epoch: %d, iter: %d, spent %.2f s, overall loss: %.3f' % (epoch, _iter, time() - begin_time, loss.item())) logger.info( 'standard acc: %.3f %%, robustness acc: %.3f %%' % (std_acc, adv_acc)) # begin_time = time() # if va_loader is not None: # va_acc, va_adv_acc = self.test(model, va_loader, True) # va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0 # logger.info('\n' + '='*30 + ' evaluation ' + '='*30) # logger.info('test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % ( # va_acc, va_adv_acc, time() - begin_time)) # logger.info('='*28 + ' end of evaluation ' + '='*28 + '\n') begin_time = time() if _iter % args.n_store_image_step == 0: tv.utils.save_image( torch.cat([data.cpu(), adv_data.cpu()], dim=0), os.path.join(args.log_folder, 'images_%d.jpg' % _iter), nrow=16) if _iter % args.n_checkpoint_step == 0: file_name = os.path.join(args.model_folder, 'checkpoint_%d.pth' % _iter) save_model(model, file_name) _iter += 1 # Update the scheduler scheduler.step() if va_loader is not None: t1 = time() va_acc, va_adv_acc = self.test(model, va_loader, self.test_attack, True) va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0 t2 = time() logger.info('\n'+'='*20 +' evaluation at epoch: %d iteration: %d '%(epoch, _iter) \ +'='*20) logger.info( 'test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % (va_acc, va_adv_acc, t2 - t1)) logger.info('=' * 28 + ' end of evaluation ' + '=' * 28 + '\n')
def train(self, model, tr_loader, va_loader=None, adv_train=False): args = self.args logger = self.logger opt = torch.optim.Adam(model.parameters(), args.learning_rate) _iter = 0 begin_time = time() for epoch in range(1, args.max_epoch + 1): for data, label in tr_loader: # print("Unique labels:", np.unique(label)) dummylabel = label data, label = tensor2cuda(data), tensor2cuda(label) if adv_train: # When training, the adversarial example is created from a random # close point to the original data point. If in evaluation mode, # just start from the original data point. adv_data = self.attack.perturb(data, label, 'mean', True) # print("Shape of Adversarial data:", adv_data.shape) # print("Shape of labels:", label.shape) # with torch.no_grad(): X_output = model(data, _eval=False, _prefinal=True) X_adv_output = model(adv_data, _eval=False, _prefinal=True) tloss = triplet_loss(X_output, X_adv_output, label) print("Triplet loss is:", tloss) print("Triplet loss RG:", tloss.requires_grad) output = model(adv_data, _eval=False) else: output = model(data, _eval=False) # Add triplet loss here ce_loss = F.cross_entropy(output, label) loss = ce_loss + tloss opt.zero_grad() loss.backward() opt.step() if _iter % args.n_eval_step == 0: if adv_train: with torch.no_grad(): stand_output = model(data, _eval=True) pred = torch.max(stand_output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 else: adv_data = self.attack.perturb(data, label, 'mean', False) with torch.no_grad(): adv_output = model(adv_data, _eval=True) pred = torch.max(adv_output, dim=1)[1] # print(label) # print(pred) adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 # only calculating the training time logger.info( 'epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' % (epoch, _iter, time() - begin_time, loss.item())) logger.info( 'standard acc: %.3f %%, robustness acc: %.3f %%' % (std_acc, adv_acc)) if va_loader is not None: va_acc, va_adv_acc = self.test(model, va_loader, True) va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0 logger.info('\n' + '=' * 30 + ' evaluation ' + '=' * 30) logger.info( 'test acc: %.3f %%, test adv acc: %.3f %%' % (va_acc, va_adv_acc)) logger.info('=' * 28 + ' end of evaluation ' + '=' * 28 + '\n') begin_time = time() if _iter % args.n_store_image_step == 0: tv.utils.save_image( torch.cat([data.cpu(), adv_data.cpu()], dim=0), os.path.join(args.log_folder, 'images_%d.jpg' % _iter), nrow=16) if _iter % args.n_checkpoint_step == 0: file_name = os.path.join(args.model_folder, 'checkpoint_%d.pth' % _iter) save_model(model, file_name) _iter += 1
def train(self, model, tr_loader, va_loader=None, adv_train=False): args = self.args logger = self.logger opt = torch.optim.Adam(model.parameters(), args.learning_rate) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, patience=400) # scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=2, gamma=0.1) _iter = 0 begin_time = time() for epoch in range(1, args.max_epoch + 1): # scheduler.step() for data, label in tr_loader: data, label = tensor2cuda(data), tensor2cuda(label) if adv_train: # When training, the adversarial example is created from a random # close point to the original data point. If in evaluation mode, # just start from the original data point. adv_data = self.attack.perturb(data, label, 'mean', True) output = model(adv_data, _eval=False) else: output = model(data, _eval=False) if self.args.nexpa_train: loss = model.loss_a(output, label) \ + model.loss_b(output, label) \ + model.loss_c(output, label) \ + model.loss_nonexpa() else: loss = F.cross_entropy(output, label) opt.zero_grad() loss.backward() opt.step() scheduler.step(loss) if _iter % args.n_eval_step == 0: if adv_train: with torch.no_grad(): stand_output = model(data, _eval=True) pred = torch.max(stand_output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 else: adv_data = self.attack.perturb(data, label, 'mean', False) with torch.no_grad(): adv_output = model(adv_data, _eval=True) pred = torch.max(adv_output, dim=1)[1] # print(label) # print(pred) adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 pred = torch.max(output, dim=1)[1] # print(pred) std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100 # only calculating the training time logger.info( 'epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' % (epoch, _iter, time() - begin_time, loss.item())) print(model.loss_a(output, label)) print(model.loss_b(output, label)) print(model.loss_c(output, label)) print(model.loss_nonexpa()) logger.info( 'standard acc: %.3f %%, robustness acc: %.3f %%' % (std_acc, adv_acc)) if va_loader is not None: va_acc, va_adv_acc = self.test(model, va_loader, True) va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0 logger.info('\n' + '=' * 30 + ' evaluation ' + '=' * 30) logger.info( 'test acc: %.3f %%, test adv acc: %.3f %%' % (va_acc, va_adv_acc)) logger.info('=' * 28 + ' end of evaluation ' + '=' * 28 + '\n') begin_time = time() if _iter % args.n_store_image_step == 0: tv.utils.save_image( torch.cat([data.cpu(), adv_data.cpu()], dim=0), os.path.join(args.log_folder, 'images_%d.jpg' % _iter), nrow=16) if _iter % args.n_checkpoint_step == 0: file_name = os.path.join(args.model_folder, 'checkpoint_%d.pth' % _iter) save_model(model, file_name) _iter += 1