def test(self, model, loader, adv_test=False):
# adv_test is False, return adv_acc as -1
total_acc = 0.0
num = 0
total_adv_acc = 0.0
with torch.no_grad():
for data, label in loader:
data, label = tensor2cuda(data), tensor2cuda(label)
output = model(data, _eval=True)
pred = torch.max(output, dim=1)[1]
te_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy(), 'sum')
total_acc += te_acc
num += output.shape[0]
if adv_test:
# use predicted label as target label
with torch.enable_grad():
adv_data = self.attack.perturb(data, pred, 'mean', False)
adv_output = model(adv_data, _eval=True)
adv_pred = torch.max(adv_output, dim=1)[1]
adv_acc = evaluate(adv_pred.cpu().numpy(), label.cpu().numpy(), 'sum')
total_adv_acc += adv_acc
else:
total_adv_acc = -num
return total_acc / num , total_adv_acc / num
def perturb(self,
original_images,
labels,
reduction4loss='mean',
random_start=False):
# original_images: values are within self.min_val and self.max_val
# The adversaries created from random close points to the original data
if random_start:
rand_perturb = torch.FloatTensor(original_images.shape).uniform_(
-self.epsilon, self.epsilon)
rand_perturb = tensor2cuda(rand_perturb)
x = original_images + rand_perturb
x.clamp_(self.min_val, self.max_val)
else:
x = original_images.clone()
x.requires_grad = True
# max_x = original_images + self.epsilon
# min_x = original_images - self.epsilon
self.model.eval()
with torch.enable_grad():
for _iter in range(self.max_iters):
outputs = self.model(x, _eval=True)
loss = F.cross_entropy(outputs,
labels,
reduction=reduction4loss)
if reduction4loss == 'none':
grad_outputs = tensor2cuda(torch.ones(loss.shape))
else:
grad_outputs = None
grads = torch.autograd.grad(loss,
x,
grad_outputs=grad_outputs,
only_inputs=True)[0]
x.data += self.alpha * torch.sign(grads.data)
# the adversaries' pixel value should within max_x and min_x due
# to the l_infinity / l2 restriction
x = project(x, original_images, self.epsilon, self._type)
# the adversaries' value should be valid pixel value
x.clamp_(self.min_val, self.max_val)
self.model.train()
return x
def generate_gradients(self, input_image, target_class):
# Put model in evaluation mode
self.model.eval()
x = input_image.clone()
x.requires_grad = True
with torch.enable_grad():
# Forward
model_output = self.model(x)
# Zero grads
self.model.zero_grad()
grad_outputs = one_hot(target_class, model_output.shape[1])
grad_outputs = tensor2cuda(grad_outputs)
grad = torch.autograd.grad(model_output,
x,
grad_outputs=grad_outputs,
only_inputs=True)[0]
self.model.train()
return grad
def cw_attack_test(model, te_loader):
model.eval()
cw_acc = 0.0
te_acc = 0.0
test_num = 0
adv_num = 0
total_te_acc = 0.0
total_cw_acc = 0.0
mean = [0]
std = [1]
inputs_box = (min((0 - m) / s for m, s in zip(mean, std)),
max((1 - m) / s for m, s in zip(mean, std)))
for data, label in te_loader:
te_acc = 0.0
data, label = tensor2cuda(data), tensor2cuda(label)
test_output = model(data, _eval=True)
test_pred = torch.max(test_output, dim=1)[1]
te_acc = evaluate(test_pred.cpu().numpy(), label.cpu().numpy(), 'sum')
total_te_acc += te_acc
test_num += test_output.shape[0]
#######################################################################
# Adversarial Samples
cw_acc = 0.0
attack = carlini_wagner_L2.L2Adversary(targeted=False,
confidence=0.0,
search_steps=10,
box=inputs_box,
optimizer_lr=5e-4)
adv_data = attack(model, data, label, to_numpy=False)
adv_output = model(adv_data.cuda(), _eval=True)
adv_num += adv_output.shape[0]
adv_pred = torch.max(adv_output, dim=1)[1]
cw_acc = evaluate(adv_pred.cpu().numpy(), label.cpu().numpy(), 'sum')
total_cw_acc += cw_acc
print("Total Test Samples", test_num)
print("Total Adversarial Samples", adv_num)
print("Accuracy before CW Attack", ((total_te_acc / test_num) * 100))
print("Accuracy after CW Attack", ((total_cw_acc / adv_num) * 100))
def train(self, model, tr_loader, va_loader=None, adv_train=False):
args = self.args
logger = self.logger
opt = torch.optim.Adam(model.parameters(), args.learning_rate, weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.MultiStepLR(opt,
milestones=[100, 150],
gamma=0.1)
_iter = 0
begin_time = time()
for epoch in range(1, args.max_epoch+1):
scheduler.step()
for data, label in tr_loader:
data, label = tensor2cuda(data), tensor2cuda(label)
if adv_train:
# When training, the adversarial example is created from a random
# close point to the original data point. If in evaluation mode,
# just start from the original data point.
adv_data = self.attack.perturb(data, label, 'mean', True)
output = model(adv_data, _eval=False)
else:
output = model(data, _eval=False)
loss = F.cross_entropy(output, label)
opt.zero_grad()
loss.backward()
opt.step()
if _iter % args.n_eval_step == 0:
t1 = time()
if adv_train:
with torch.no_grad():
stand_output = model(data, _eval=True)
pred = torch.max(stand_output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100
else:
adv_data = self.attack.perturb(data, label, 'mean', False)
with torch.no_grad():
adv_output = model(adv_data, _eval=True)
pred = torch.max(adv_output, dim=1)[1]
# print(label)
# print(pred)
adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100
t2 = time()
print('%.3f' % (t2 - t1))
logger.info('epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' % (
epoch, _iter, time()-begin_time, loss.item()))
logger.info('standard acc: %.3f %%, robustness acc: %.3f %%' % (
std_acc, adv_acc))
# begin_time = time()
# if va_loader is not None:
# va_acc, va_adv_acc = self.test(model, va_loader, True)
# va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0
# logger.info('\n' + '='*30 + ' evaluation ' + '='*30)
# logger.info('test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % (
# va_acc, va_adv_acc, time() - begin_time))
# logger.info('='*28 + ' end of evaluation ' + '='*28 + '\n')
begin_time = time()
if _iter % args.n_store_image_step == 0:
tv.utils.save_image(torch.cat([data.cpu(), adv_data.cpu()], dim=0),
os.path.join(args.log_folder, 'images_%d.jpg' % _iter),
nrow=16)
if _iter % args.n_checkpoint_step == 0:
file_name = os.path.join(args.model_folder, 'checkpoint_%d.pth' % _iter)
save_model(model, file_name)
_iter += 1
if va_loader is not None:
t1 = time()
va_acc, va_adv_acc = self.test(model, va_loader, True)
va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0
t2 = time()
logger.info('\n'+'='*20 +' evaluation at epoch: %d iteration: %d '%(epoch, _iter) \
+'='*20)
logger.info('test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % (
va_acc, va_adv_acc, t2-t1))
logger.info('='*28+' end of evaluation '+'='*28+'\n')
def train(self, model, tr_loader, va_loader=None, adv_train=False):
args = self.args
logger = self.logger
best_adv_acc = 0.
opt = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=0.9,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.MultiStepLR(opt,
milestones=[75, 90],
gamma=0.1)
_iter = 0
for epoch in range(1, args.max_epoch + 1):
begin_time = time()
scheduler.step()
for data, label in tr_loader:
data, label = tensor2cuda(data), tensor2cuda(label)
if adv_train:
# When training, the adversarial example is created from a random
# close point to the original data point. If in evaluation mode,
# just start from the original data point.
adv_data = self.attack.perturb(data, label, 'mean', True,
args.beta)
output, class_wise_output = model(adv_data,
y=label,
_eval=False)
loss = F.cross_entropy(output, label)
channel_reg_loss = 0.
for extra_output in class_wise_output:
channel_reg_loss += F.cross_entropy(
extra_output, label)
channel_reg_loss = channel_reg_loss / len(
class_wise_output)
loss = loss + args.beta * channel_reg_loss
else:
output, class_wise_output = model(data,
y=label,
_eval=False)
loss = F.cross_entropy(output, label)
channel_reg_loss = 0.
for extra_output in class_wise_output:
channel_reg_loss += F.cross_entropy(
extra_output, label)
channel_reg_loss = channel_reg_loss / len(
class_wise_output)
loss = loss + args.beta * channel_reg_loss
opt.zero_grad()
loss.backward()
opt.step()
if _iter % args.n_eval_step == 0:
if adv_train:
with torch.no_grad():
stand_output, _ = model(data, _eval=True)
pred = torch.max(stand_output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
adv_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
else:
adv_data = self.attack.perturb(data, label, 'mean',
False)
with torch.no_grad():
adv_output, _ = model(adv_data, _eval=True)
pred = torch.max(adv_output, dim=1)[1]
adv_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
logger.info(
'epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' %
(epoch, _iter, time() - begin_time, loss.item()))
logger.info(
'standard acc: %.3f %%, robustness acc: %.3f %%' %
(std_acc, adv_acc))
if _iter % args.n_checkpoint_step == 0:
file_name = os.path.join(args.model_folder,
'checkpoint_last.pth')
save_model(model, file_name)
_iter += 1
epoch_cost = time()
print('cost:', epoch_cost - begin_time)
# assert False
if va_loader is not None:
t1 = time()
va_acc, va_adv_acc = self.test(model, va_loader, True, False)
va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0
t2 = time()
logger.info('\n' + '=' * 20 + ' evaluation at epoch: %d iteration: %d ' % (epoch, _iter) \
+ '=' * 20)
logger.info(
'test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' %
(va_acc, va_adv_acc, t2 - t1))
logger.info('=' * 28 + ' end of evaluation ' + '=' * 28 + '\n')
if va_adv_acc > best_adv_acc:
file_name = os.path.join(args.model_folder,
'checkpoint_best_adv.pth')
save_model(model, file_name)
best_adv_acc = va_adv_acc
def train(self, model, tr_loader, va_loader=None, adv_train=False):
args = self.args
logger = self.logger
opt = torch.optim.Adam(model.parameters(), args.learning_rate, weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[100, 200], gamma=0.1)
_iter = 0
begin_time = time()
best_loss = 999
for epoch in range(1, args.max_epoch+1):
for data, label in tr_loader:
# for sample in tr_loader:
# data, label = sample['buffers'], sample['labels']
# print('data shape is ', data.shape)
# print('label shape is ', label.shape)
data, label = tensor2cuda(data), tensor2cuda(label)
if adv_train:
# When training, the adversarial example is created from a random
# close point to the original data point. If in evaluation mode,
# just start from the original data point.
adv_data = self.attack.perturb(data, label, 'mean', True)
model.train()
output = model(adv_data)
else:
model.train()
output = model(data)
loss = F.binary_cross_entropy(torch.sigmoid(output), label)
opt.zero_grad()
loss.backward()
opt.step()
t = Variable(torch.Tensor([0.5]).cuda()) # threshold to compute accuracy
if _iter % args.n_eval_step == 0:
t1 = time()
if adv_train:
with torch.no_grad():
model.eval()
stand_output = model(data)
# pred = torch.max(stand_output, dim=1)[1] # this give us the indices tensor
pred = torch.sigmoid(stand_output)
out = (pred > t).float()
# print(pred.shape)
# print(out.shape)
# std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100
stdacc_list = evaluate_(out.cpu().numpy(), label.cpu().numpy())
# print('std accuracy list shape: ', np.array(stdacc_list).shape)
# pred = torch.max(output, dim=1)[1]
pred = torch.sigmoid(output)
out = (pred > t).float()
# print(pred)
# adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100
advacc_list = evaluate_(out.cpu().numpy(), label.cpu().numpy())
# print('adv accuracy list shape: ', np.array(stdacc_list).shape)
else:
adv_data = self.attack.perturb(data, label, 'mean', False)
with torch.no_grad():
model.eval()
adv_output = model(adv_data)
# adv_output = model(adv_data, _eval=True)
# pred = torch.max(adv_output, dim=1)[1]
pred = torch.sigmoid(adv_output)
out = (pred > t).float()
# print(label)
# print(pred)
# adv_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100
advacc_list = evaluate_(out.cpu().numpy(), label.cpu().numpy())
# pred = torch.max(output, dim=1)[1]
pred = torch.sigmoid(output)
out = (pred > t).float()
# print(pred)
# std_acc = evaluate(pred.cpu().numpy(), label.cpu().numpy()) * 100
stdacc_list = evaluate_(out.cpu().numpy(), label.cpu().numpy())
t2 = time()
print('%.3f' % (t2 - t1))
logger.info('epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' % (
epoch, _iter, time()-begin_time, loss.item()))
begin_time = time()
if _iter % args.n_checkpoint_step == 0:
file_name = os.path.join(args.model_folder, 'checkpoint_%d.pth' % _iter)
save_model(model, file_name)
_iter += 1
scheduler.step()
if va_loader is not None:
t1 = time()
va_acc, va_adv_acc, va_stdloss, va_advloss = self.test(model, va_loader, True, False)
va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0
if va_stdloss < best_loss:
best_loss = va_stdloss
file_name = os.path.join(args.model_folder, 'checkpoint_best.pth')
save_model(model, file_name)
t2 = time()
logger.info('\n'+'='*20 +' evaluation at epoch: %d iteration: %d '%(epoch, _iter) \
+'='*20)
logger.info('test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % (
va_acc, va_adv_acc, t2-t1))
logger.info('test loss: %.3f , test adv loss: %.3f , spent: %.3f' % (
va_stdloss, va_advloss, t2-t1))
logger.info('='*28+' end of evaluation '+'='*28+'\n')
def test(self, model, loader, adv_test=False,
use_pseudo_label=False, if_AUC=False):
# adv_test is False, return adv_acc as -1
total_acc = 0.0
num = 0
total_adv_acc = 0.0
total_stdloss = 0.0
total_advloss = 0.0
t = Variable(torch.Tensor([0.5]).cuda()) # threshold to compute accuracy
label_list = []
pred_list = []
predadv_list = []
with torch.no_grad():
for data, label in loader:
# for sample in loader:
# data, label = sample['buffers'], sample['labels']
data, label = tensor2cuda(data), tensor2cuda(label)
model.eval()
output = model(data)
std_loss = F.binary_cross_entropy(torch.sigmoid(output), label)
pred = torch.sigmoid(output)
out = (pred > t).float()
te_acc = np.mean(evaluate_(out.cpu().numpy(), label.cpu().numpy()))
total_acc += te_acc
total_stdloss += std_loss
if if_AUC:
label_list.append(label.cpu().numpy())
pred_list.append(pred.cpu().numpy())
# num += output.shape[0]
num += 1
if adv_test:
# use predicted label as target label
with torch.enable_grad():
adv_data = self.attack.perturb(data,
pred if use_pseudo_label else label,
'mean', False)
model.eval()
adv_output = model(adv_data)
adv_loss = F.binary_cross_entropy(torch.sigmoid(adv_output), label)
adv_pred = torch.sigmoid(adv_output)
if if_AUC:
predadv_list.append(adv_pred.cpu().numpy())
adv_out = (adv_pred > t).float()
adv_acc = np.mean(evaluate_(adv_out.cpu().numpy(), label.cpu().numpy()))
total_adv_acc += adv_acc
total_advloss += adv_loss
else:
total_adv_acc = -num
if if_AUC:
pred = np.squeeze(np.array(pred_list))
predadv = np.squeeze(np.array(predadv_list))
label = np.squeeze(np.array(label_list))
np.save(os.path.join(self.args.log_folder, 'y_pred.npy'), pred)
# np.save(os.path.join(self.args.log_folder, 'y_predadv_'+str(args.epsilon)+'.npy'), predadv)
np.save(os.path.join(self.args.log_folder, 'y_true.npy'), label)
# PRED_label = ['No Finding', 'Cardiomegaly', 'Edema',
# 'Consolidation', 'Pneumonia', 'Atelectasis',
# 'Pneumothorax', 'Pleural Effusion']
# PRED_label = ['healthy', 'partially injured', 'completely ruptured']
# PRED_label = ['malignancy']
# plot_AUC(pred, label, self.args.log_folder, 'auc.png', PRED_label)
# plot_AUC(predadv, label, self.args.log_folder, 'auc_'+str(args.epsilon)+'.png', PRED_label)
# np.save('predstd_'+str(args.epsilon)+'.npy', pred)
# np.save('predstdadv_'+str(args.epsilon)+'.npy', predadv)
# np.save('labelstd_'+str(args.epsilon)+'.npy', label)
else:
return total_acc / num, total_adv_acc / num, total_stdloss / num, total_advloss / num
label_dict = LabelDict(args.dataset)
te_dataset = tv.datasets.CIFAR10(args.data_root,
train=False,
transform=tv.transforms.ToTensor(),
download=True)
te_loader = DataLoader(te_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
for data, label in te_loader:
data, label = tensor2cuda(data), tensor2cuda(label)
break
adv_list = []
pred_list = []
with torch.no_grad():
model = WideResNet(depth=34, num_classes=10, widen_factor=10, dropRate=0.0)
load_model(model, args.load_checkpoint)
if torch.cuda.is_available():
model.cuda()
def test(self,
model,
loader,
epsilon,
adv_test=False,
use_pseudo_label=False,
if_AUC=False):
# adv_test is False, return adv_acc as -1
total_acc = 0.0
num = 0
total_adv_acc = 0.0
total_stdloss = 0.0
total_advloss = 0.0
t = Variable(torch.Tensor([0.5
]).cuda()) # threshold to compute accuracy
label_list = []
pred_list = []
predadv_list = []
with torch.no_grad():
for data, label in loader:
data, label = tensor2cuda(data), tensor2cuda(label)
model.eval()
output = model(data, [0])
std_loss = F.binary_cross_entropy(torch.sigmoid(output), label)
pred = torch.sigmoid(output)
out = (pred > t).float()
te_acc = np.mean(
evaluate_(out.cpu().numpy(),
label.cpu().numpy()))
total_acc += te_acc
total_stdloss += std_loss
if if_AUC:
label_list.append(label.cpu().numpy())
pred_list.append(pred.cpu().numpy())
# num += output.shape[0]
num += 1
if adv_test:
# use predicted label as target label
with torch.enable_grad():
adv_data = self.attack.perturb(
data, pred if use_pseudo_label else label, 'mean',
False, True)
model.eval()
adv_output = model(adv_data, [1])
adv_loss = F.binary_cross_entropy(
torch.sigmoid(adv_output), label)
adv_pred = torch.sigmoid(adv_output)
if if_AUC:
predadv_list.append(adv_pred.cpu().numpy())
adv_out = (adv_pred > t).float()
adv_acc = np.mean(
evaluate_(adv_out.cpu().numpy(),
label.cpu().numpy()))
total_adv_acc += adv_acc
total_advloss += adv_loss
else:
total_adv_acc = -num
if if_AUC:
pred = np.squeeze(np.array(pred_list))
predadv = np.squeeze(np.array(predadv_list))
label = np.squeeze(np.array(label_list))
np.save(os.path.join(self.log_folder, 'y_pred.npy'), pred)
np.save(os.path.join(self.log_folder, 'y_true.npy'), label)
np.save(
os.path.join(self.log_folder,
'y_predadv_' + str(epsilon) + '.npy'), predadv)
else:
return total_acc / num, total_adv_acc / num, total_stdloss / num, total_advloss / num
def train(self, model, tr_loader, va_loader=None, adv_train=False):
args = self.args
logger = self.logger
opt = torch.optim.Adam(model.parameters(),
args.learning_rate,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.MultiStepLR(
opt, milestones=[50, 100, 150], gamma=0.1)
_iter = 0
begin_time = time()
# Loss mixer: (ce_loss_orig + (lambda1 * ce_loss_adversarial))/(1 + lambda1) + (lambda2 * custom_loss)
lambda1 = 0.6
if args.custom_regularizer == 'Triplet': # 1 for triplet
lambda2 = 1
elif args.custom_regularizer == 'Npairs': # 0.5 for npairs for now
lambda2 = 0.5
else:
lambda2 = 0
for epoch in range(1, args.max_epoch + 1):
#scheduler.step()
for data, label in tr_loader:
data, label = tensor2cuda(data), tensor2cuda(label)
if adv_train:
# Outputs from original samples
output = model(data, _eval=False)
# When training, the adversarial example is created from a random
# close point to the original data point. If in evaluation mode,
# just start from the original data point.
if args.adv_train_mode == 'FGSM':
adv_data = self.train_attack.perturb(
data, label, 'mean', True)
elif args.adv_train_mode == 'CW':
adv_data = self.train_attack(model,
data,
label,
to_numpy=False)
adv_data = adv_data.cuda()
# Outputs from adversarial samples
adv_output = model(adv_data, _eval=False)
# Shape of pred: batch_size
# Predicted labels on adversarial samples
pred = torch.max(adv_output, dim=1)[1]
if args.custom_regularizer == 'No_reg':
tloss = 0
else:
# Shape of output: batch_size * (64*widen_factor)
X_output = model(data, _eval=False, _prefinal=True)
X_adv_output = model(adv_data,
_eval=False,
_prefinal=True)
if args.custom_regularizer == 'Triplet':
tloss = triplet_loss(X_output, X_adv_output, label,
pred, logger)
elif args.custom_regularizer == 'Npairs':
tloss = npairs_loss(X_output, X_adv_output, label)
else:
output = model(data, _eval=False)
tloss = 0
ce_loss = F.cross_entropy(output, label)
if adv_train:
# Adding loss due to adversarial samples in the mix
ce_loss = ce_loss + lambda1 * F.cross_entropy(
adv_output, label)
ce_loss = ce_loss / (1 + lambda1)
loss = ce_loss + lambda2 * tloss
opt.zero_grad()
loss.backward()
opt.step()
if _iter % args.n_eval_step == 0:
t1 = time()
logger.info(
'Total Loss logger: %.3f, CE Loss: %.3f, %s loss: %.3f'
% (loss, ce_loss, args.custom_regularizer, tloss))
if adv_train:
with torch.no_grad():
stand_output = model(data, _eval=True)
pred_stand = torch.max(stand_output, dim=1)[1]
std_acc = evaluate(pred_stand.cpu().numpy(),
label.cpu().numpy()) * 100
adv_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
else:
if args.adv_train_mode == 'FGSM':
adv_data = self.train_attack.perturb(
data, label, 'mean', False)
elif args.adv_train_mode == 'CW':
adv_data = self.train_attack(model,
data,
label,
to_numpy=False)
adv_data = adv_data.cuda()
with torch.no_grad():
adv_output = model(adv_data, _eval=True)
pred = torch.max(adv_output, dim=1)[1]
adv_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
t2 = time()
print('%.3f' % (t2 - t1))
logger.info(
'epoch: %d, iter: %d, spent %.2f s, overall loss: %.3f'
% (epoch, _iter, time() - begin_time, loss.item()))
logger.info(
'standard acc: %.3f %%, robustness acc: %.3f %%' %
(std_acc, adv_acc))
# begin_time = time()
# if va_loader is not None:
# va_acc, va_adv_acc = self.test(model, va_loader, True)
# va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0
# logger.info('\n' + '='*30 + ' evaluation ' + '='*30)
# logger.info('test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' % (
# va_acc, va_adv_acc, time() - begin_time))
# logger.info('='*28 + ' end of evaluation ' + '='*28 + '\n')
begin_time = time()
if _iter % args.n_store_image_step == 0:
tv.utils.save_image(
torch.cat([data.cpu(), adv_data.cpu()], dim=0),
os.path.join(args.log_folder, 'images_%d.jpg' % _iter),
nrow=16)
if _iter % args.n_checkpoint_step == 0:
file_name = os.path.join(args.model_folder,
'checkpoint_%d.pth' % _iter)
save_model(model, file_name)
_iter += 1
# Update the scheduler
scheduler.step()
if va_loader is not None:
t1 = time()
va_acc, va_adv_acc = self.test(model, va_loader,
self.test_attack, True)
va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0
t2 = time()
logger.info('\n'+'='*20 +' evaluation at epoch: %d iteration: %d '%(epoch, _iter) \
+'='*20)
logger.info(
'test acc: %.3f %%, test adv acc: %.3f %%, spent: %.3f' %
(va_acc, va_adv_acc, t2 - t1))
logger.info('=' * 28 + ' end of evaluation ' + '=' * 28 + '\n')
def train(self, model, tr_loader, va_loader=None, adv_train=False):
args = self.args
logger = self.logger
opt = torch.optim.Adam(model.parameters(), args.learning_rate)
_iter = 0
begin_time = time()
for epoch in range(1, args.max_epoch + 1):
for data, label in tr_loader:
# print("Unique labels:", np.unique(label))
dummylabel = label
data, label = tensor2cuda(data), tensor2cuda(label)
if adv_train:
# When training, the adversarial example is created from a random
# close point to the original data point. If in evaluation mode,
# just start from the original data point.
adv_data = self.attack.perturb(data, label, 'mean', True)
# print("Shape of Adversarial data:", adv_data.shape)
# print("Shape of labels:", label.shape)
# with torch.no_grad():
X_output = model(data, _eval=False, _prefinal=True)
X_adv_output = model(adv_data, _eval=False, _prefinal=True)
tloss = triplet_loss(X_output, X_adv_output, label)
print("Triplet loss is:", tloss)
print("Triplet loss RG:", tloss.requires_grad)
output = model(adv_data, _eval=False)
else:
output = model(data, _eval=False)
# Add triplet loss here
ce_loss = F.cross_entropy(output, label)
loss = ce_loss + tloss
opt.zero_grad()
loss.backward()
opt.step()
if _iter % args.n_eval_step == 0:
if adv_train:
with torch.no_grad():
stand_output = model(data, _eval=True)
pred = torch.max(stand_output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
adv_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
else:
adv_data = self.attack.perturb(data, label, 'mean',
False)
with torch.no_grad():
adv_output = model(adv_data, _eval=True)
pred = torch.max(adv_output, dim=1)[1]
# print(label)
# print(pred)
adv_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
# only calculating the training time
logger.info(
'epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' %
(epoch, _iter, time() - begin_time, loss.item()))
logger.info(
'standard acc: %.3f %%, robustness acc: %.3f %%' %
(std_acc, adv_acc))
if va_loader is not None:
va_acc, va_adv_acc = self.test(model, va_loader, True)
va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0
logger.info('\n' + '=' * 30 + ' evaluation ' +
'=' * 30)
logger.info(
'test acc: %.3f %%, test adv acc: %.3f %%' %
(va_acc, va_adv_acc))
logger.info('=' * 28 + ' end of evaluation ' +
'=' * 28 + '\n')
begin_time = time()
if _iter % args.n_store_image_step == 0:
tv.utils.save_image(
torch.cat([data.cpu(), adv_data.cpu()], dim=0),
os.path.join(args.log_folder, 'images_%d.jpg' % _iter),
nrow=16)
if _iter % args.n_checkpoint_step == 0:
file_name = os.path.join(args.model_folder,
'checkpoint_%d.pth' % _iter)
save_model(model, file_name)
_iter += 1
def train(self, model, tr_loader, va_loader=None, adv_train=False):
args = self.args
logger = self.logger
opt = torch.optim.Adam(model.parameters(), args.learning_rate)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt,
patience=400)
# scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=2, gamma=0.1)
_iter = 0
begin_time = time()
for epoch in range(1, args.max_epoch + 1):
# scheduler.step()
for data, label in tr_loader:
data, label = tensor2cuda(data), tensor2cuda(label)
if adv_train:
# When training, the adversarial example is created from a random
# close point to the original data point. If in evaluation mode,
# just start from the original data point.
adv_data = self.attack.perturb(data, label, 'mean', True)
output = model(adv_data, _eval=False)
else:
output = model(data, _eval=False)
if self.args.nexpa_train:
loss = model.loss_a(output, label) \
+ model.loss_b(output, label) \
+ model.loss_c(output, label) \
+ model.loss_nonexpa()
else:
loss = F.cross_entropy(output, label)
opt.zero_grad()
loss.backward()
opt.step()
scheduler.step(loss)
if _iter % args.n_eval_step == 0:
if adv_train:
with torch.no_grad():
stand_output = model(data, _eval=True)
pred = torch.max(stand_output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
adv_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
else:
adv_data = self.attack.perturb(data, label, 'mean',
False)
with torch.no_grad():
adv_output = model(adv_data, _eval=True)
pred = torch.max(adv_output, dim=1)[1]
# print(label)
# print(pred)
adv_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
pred = torch.max(output, dim=1)[1]
# print(pred)
std_acc = evaluate(pred.cpu().numpy(),
label.cpu().numpy()) * 100
# only calculating the training time
logger.info(
'epoch: %d, iter: %d, spent %.2f s, tr_loss: %.3f' %
(epoch, _iter, time() - begin_time, loss.item()))
print(model.loss_a(output, label))
print(model.loss_b(output, label))
print(model.loss_c(output, label))
print(model.loss_nonexpa())
logger.info(
'standard acc: %.3f %%, robustness acc: %.3f %%' %
(std_acc, adv_acc))
if va_loader is not None:
va_acc, va_adv_acc = self.test(model, va_loader, True)
va_acc, va_adv_acc = va_acc * 100.0, va_adv_acc * 100.0
logger.info('\n' + '=' * 30 + ' evaluation ' +
'=' * 30)
logger.info(
'test acc: %.3f %%, test adv acc: %.3f %%' %
(va_acc, va_adv_acc))
logger.info('=' * 28 + ' end of evaluation ' +
'=' * 28 + '\n')
begin_time = time()
if _iter % args.n_store_image_step == 0:
tv.utils.save_image(
torch.cat([data.cpu(), adv_data.cpu()], dim=0),
os.path.join(args.log_folder, 'images_%d.jpg' % _iter),
nrow=16)
if _iter % args.n_checkpoint_step == 0:
file_name = os.path.join(args.model_folder,
'checkpoint_%d.pth' % _iter)
save_model(model, file_name)
_iter += 1
