def evaluate(loader, model, apply_func, log, verbose):
batch_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
end = time.time()
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda().long()
if apply_func is not None:
X, y = apply_func(model, X, y)
if y.dim() == 2:
y = y.squeeze(1)
with torch.no_grad():
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.max(1)[1] != y).float().sum() / X.size(0)
# measure accuracy and record loss
losses.update(ce.item(), X.size(0))
errors.update(err.item(), X.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# print(i, ce.item(), err.item(), file=log)
if verbose:
endline = '\n' if i % verbose == 0 else '\r'
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {errors.val:.3f} ({errors.avg:.3f})'.format(
i,
len(loader),
batch_time=batch_time,
loss=losses,
errors=errors),
end=endline)
log.flush()
del X, y, out, ce, err
if DEBUG and i == 10:
break
return losses.avg, errors.avg
def evaluate_robust(loader, model, epsilon, log, verbose):
batch_time = AverageMeter()
robust_losses = AverageMeter()
robust_errors = AverageMeter()
end = time.time()
torch.set_grad_enabled(False)
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda().long()
if y.dim() == 2:
y = y.squeeze(1)
robust_ce, robust_err = robust_loss(model, epsilon, X, y)
# measure accuracy and record loss
robust_losses.update(robust_ce.item(), X.size(0))
robust_errors.update(robust_err, X.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# print(i, robust_ce.item(), robust_err, file=log)
if verbose:
endline = '\n' if i % verbose == 0 else '\r'
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Robust loss {rloss.val:.3f} ({rloss.avg:.3f})\t'
'Robust error {rerrors.val:.3f} ({rerrors.avg:.3f})'.format(
i,
len(loader),
batch_time=batch_time,
rloss=robust_losses,
rerrors=robust_errors),
end=endline)
log.flush()
del X, y, robust_ce
if DEBUG and i == 10:
break
torch.set_grad_enabled(True)
torch.cuda.empty_cache()
return robust_losses.avg, robust_errors.avg
def evaluate_transfer_robust(loader,
t_model,
model,
epsilon,
adaptive_vp_rate,
log,
verbose,
real_time=False,
evaluate=False,
clip_grad=None,
**kwargs):
batch_time = AverageMeter()
robust_losses = AverageMeter()
robust_errors = AverageMeter()
vp_rates = AverageMeter()
invp_rates = AverageMeter()
end = time.time()
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda().long()
if y.dim() == 2:
y = y.squeeze(1)
robust_ce, robust_err, _, _, v_point_rate, eta, inv_point_rate = \
robust_loss_transfer(model, t_model, epsilon,
Variable(X), Variable(y), **kwargs)
# measure accuracy and record loss
robust_losses.update(robust_ce.detach().item(), X.size(0))
robust_errors.update(robust_err, X.size(0))
vp_rates.update(v_point_rate, X.size(0))
invp_rates.update(inv_point_rate, X.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# print(i, robust_ce.detach().item(), robust_err, v_point_rate, inv_point_rate, file=log)
if verbose:
endline = '\n' if i % verbose == 0 else '\r'
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'T Robust loss {rloss.val:.4f} ({rloss.avg:.4f})\t'
'T Robust error {rerrors.val:.3f} ({rerrors.avg:.3f})\t'
'VP Rate {vp_rate.val: .3f} ({vp_rate.avg:.3f})\t'
'INVP Rate {invp_rate.val: .3f} ({invp_rate.avg:.3f})\t'.
format(i,
len(loader),
batch_time=batch_time,
vp_rate=vp_rates,
invp_rate=invp_rates,
rloss=robust_losses,
rerrors=robust_errors),
end=endline)
log.flush()
del X, y, robust_ce, robust_err
if DEBUG and i == 10:
break
torch.cuda.empty_cache()
return robust_losses.avg, robust_errors.avg, vp_rates.avg, invp_rates.avg
def test_transferability_subset(loader, attack_method, epsilon, torch_model1,
torch_model2, verbose, batch_size):
batch_time = AverageMeter()
err12s = AverageMeter()
err21s = AverageMeter()
end = time.time()
sess = tf.Session(config=config)
x_op = tf.placeholder(tf.float32, shape=(
None,
1,
28,
28,
))
# Convert pytorch model to a tf_model and wrap it in cleverhans
tf_model_fn1 = convert_pytorch_model_to_tf(torch_model1)
tf_model_fn2 = convert_pytorch_model_to_tf(torch_model2)
# Attack Parameters
if attack_method == 'CW':
params = {
'binary_search_steps': 1,
# 'y': None,
'max_iterations': CW_ATTACK_ITERATIONS,
'learning_rate': CW_LEARNING_RATE,
'batch_size': batch_size,
'initial_const': 10
}
elif attack_method == 'PGD':
params = {
'eps': epsilon,
'clip_min': 0.,
'clip_max': 1.,
'eps_iter': 0.005,
'nb_iter': 100,
'rand_init': False
}
elif attack_method == 'FGSM':
params = {'eps': epsilon, 'clip_min': 0., 'clip_max': 1.}
else:
raise Exception('Unknown attack method %s'.format(attack_method))
# Model1 --> Model2
cleverhans_model1 = CallableModelWrapper(tf_model_fn1,
output_layer='logits')
cleverhans_model2 = CallableModelWrapper(tf_model_fn2,
output_layer='logits')
# Create an attack
if attack_method == 'CW':
attk1 = CarliniWagnerL2(cleverhans_model1, sess=sess)
if attack_method == 'PGD':
attk1 = ProjectedGradientDescent(cleverhans_model1, sess=sess)
if attack_method == 'FGSM':
attk1 = FastGradientMethod(cleverhans_model1, sess=sess)
if attack_method == 'CW':
attk2 = CarliniWagnerL2(cleverhans_model2, sess=sess)
if attack_method == 'PGD':
attk2 = ProjectedGradientDescent(cleverhans_model2, sess=sess)
if attack_method == 'FGSM':
attk2 = FastGradientMethod(cleverhans_model2, sess=sess)
adv_x_op1 = attk1.generate(x_op, **params)
adv_x_op2 = attk2.generate(x_op, **params)
# Test on model1 and model2
adv_preds_op11 = tf_model_fn1(adv_x_op1)
adv_preds_op12 = tf_model_fn2(adv_x_op1)
adv_preds_op21 = tf_model_fn1(adv_x_op2)
adv_preds_op22 = tf_model_fn2(adv_x_op2)
for i, (xs, ys) in enumerate(loader):
(adv_preds11, adv_preds12) = sess.run((adv_preds_op11, adv_preds_op12),
feed_dict={x_op: xs})
(adv_preds21, adv_preds22) = sess.run((adv_preds_op21, adv_preds_op22),
feed_dict={x_op: xs})
cnt11 = int((np.argmax(adv_preds11, axis=1) != ys).sum())
cnt22 = int((np.argmax(adv_preds22, axis=1) != ys).sum())
if cnt11 > 0:
err12 = float(
((np.argmax(adv_preds12, axis=1) != ys) *
(np.argmax(adv_preds11, axis=1) != ys)).sum()) / float(cnt11)
err12s.update(err12, cnt11)
if cnt22 > 0:
err21 = float(
((np.argmax(adv_preds22, axis=1) != ys) *
(np.argmax(adv_preds21, axis=1) != ys)).sum()) / float(cnt22)
err21s.update(err21, cnt22)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if verbose:
endline = '\n' if i % verbose == 0 else '\r'
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'error 1->2 {err12.val:.3f} ({err12.avg:.3f})\t'
'error 2->1 {err21.val:.3f} ({err21.avg:.3f})\t'.format(
i,
len(loader),
batch_time=batch_time,
err12=err12s,
err21=err21s),
end=endline)
sess.close()
return err12s.avg, err21s.avg
def evaluate_trans(loader,
dataset,
model,
epoch,
epsilon,
ref_model,
clip_min=0.,
clip_max=1.,
eps_iter=0.005,
nb_iter=100,
rand_init=False,
verbose=20):
batch_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
params = {
'eps': epsilon,
'clip_min': clip_min,
'clip_max': clip_max,
'eps_iter': eps_iter,
'nb_iter': nb_iter,
'rand_init': rand_init
}
sess = tf.Session(config=config)
x_op = tf.placeholder(tf.float32, shape=(
None,
1,
28,
28,
))
model.eval()
ref_model.eval()
tf_model = convert_pytorch_model_to_tf(ref_model)
cleverhans_model = CallableModelWrapper(tf_model, output_layer='logits')
attk = ProjectedGradientDescent(cleverhans_model, sess=sess)
adv_x_op = attk.generate(x_op, **params)
end = time.time()
for i, (X, y) in enumerate(loader):
X_adv = sess.run((adv_x_op), feed_dict={x_op: X})
X, y = Variable(torch.tensor(X_adv)).cuda(), y.cuda()
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.data.max(1)[1] != y).float().sum() / X.size(0)
# measure accuracy and record loss
losses.update(ce.item(), X.size(0))
errors.update(err.item(), X.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
endline = '\n' if i % verbose == 0 else '\r'
print('Adv test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {error.val:.3f} ({error.avg:.3f})'.format(
i,
len(loader),
batch_time=batch_time,
loss=losses,
error=errors),
end=endline)
if DEBUG and i == 10:
break
print('\n * Error {error.avg:.3f}'.format(error=errors))
return losses.avg, errors.avg
def evaluate_clean(loader, model, epoch, verbose=20):
batch_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
model.eval()
end = time.time()
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda()
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.data.max(1)[1] != y).float().sum() / X.size(0)
# measure accuracy and record loss
losses.update(ce.item(), X.size(0))
errors.update(err.item(), X.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
endline = '\n' if i % verbose == 0 else '\r'
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {error.val:.3f} ({error.avg:.3f})'.format(
i,
len(loader),
batch_time=batch_time,
loss=losses,
error=errors),
end=endline)
if DEBUG and i == 10:
break
print('\n * Error {error.avg:.3f}'.format(error=errors))
return losses.avg, errors.avg
def trans_reg_train(loader,
model,
opt,
epoch,
epsilon,
ref_model,
lbda,
verbose=20):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
dps = AverageMeter()
model.train()
ref_model.train()
end = time.time()
for i, (X, y) in enumerate(loader):
X, y = Variable(X.cuda(), requires_grad=True), Variable(y).cuda()
data_time.update(time.time() - end)
pred2 = ref_model(X)
loss2 = F.cross_entropy(pred2, y)
(grad2, ) = grad(loss2, X, create_graph=True)
grad2 = grad2.view(len(X), -1)
grad2 = F.normalize(grad2)
grad2 = torch.tensor(grad2.detach().cpu().numpy()).cuda()
pred1 = model(X)
loss1 = F.cross_entropy(pred1, y)
(grad1, ) = grad(loss1, X, create_graph=True)
grad1 = grad1.view(len(X), -1)
grad1 = F.normalize(grad1)
X.requires_grad_(False)
dp = torch.sum(torch.mul(grad1, grad2), dim=1)
dp = torch.mean(dp)
loss = loss1 + dp * lbda
opt.zero_grad()
loss.backward()
opt.step()
err = (pred1.data.max(1)[1] != y).float().sum() / X.size(0)
batch_time.update(time.time() - end)
end = time.time()
losses.update(loss.item(), X.size(0))
errors.update(err.item(), X.size(0))
dps.update(dp, X.size(0))
if verbose and i % verbose == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {errors.val:.3f} ({errors.avg:.3f})\t'
'Dp {dps.val:.3f} ({dps.avg:.3f})'.format(
epoch,
i,
len(loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
errors=errors,
dps=dps))
if DEBUG and i == 10:
break
return losses.avg, errors.avg, dps.avg
def trans_train(loader,
model,
opt,
epoch,
epsilon,
ref_model,
clip_min=0.,
clip_max=1.,
eps_iter=0.005,
nb_iter=100,
rand_init=False,
verbose=20):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
model.train()
params = {
'eps': epsilon,
'clip_min': clip_min,
'clip_max': clip_max,
'eps_iter': eps_iter,
'nb_iter': nb_iter,
'rand_init': rand_init
}
sess = tf.Session(config=config)
x_op = tf.placeholder(tf.float32, shape=(
None,
1,
28,
28,
))
tf_model = convert_pytorch_model_to_tf(ref_model)
cleverhans_model = CallableModelWrapper(tf_model, output_layer='logits')
attk = ProjectedGradientDescent(cleverhans_model, sess=sess)
adv_x_op = attk.generate(x_op, **params)
end = time.time()
for i, (X, y) in enumerate(loader):
X_adv = sess.run((adv_x_op), feed_dict={x_op: X})
X, y = Variable(torch.tensor(X_adv)).cuda(), y.cuda()
data_time.update(time.time() - end)
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.data.max(1)[1] != y).float().sum() / X.size(0)
opt.zero_grad()
ce.backward()
opt.step()
batch_time.update(time.time() - end)
end = time.time()
losses.update(ce.item(), X.size(0))
errors.update(err.item(), X.size(0))
if verbose and i % verbose == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {errors.val:.3f} ({errors.avg:.3f})'.format(
epoch,
i,
len(loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
errors=errors))
if DEBUG and i == 10:
break
return losses.avg, errors.avg
def train(loader, model, opt, epoch, verbose):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
model.train()
end = time.time()
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda()
data_time.update(time.time() - end)
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.data.max(1)[1] != y).float().sum() / X.size(0)
opt.zero_grad()
ce.backward()
opt.step()
batch_time.update(time.time() - end)
end = time.time()
losses.update(ce.item(), X.size(0))
errors.update(err.item(), X.size(0))
if verbose and i % verbose == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {errors.val:.3f} ({errors.avg:.3f})'.format(
epoch,
i,
len(loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
errors=errors))
if DEBUG and i == 10:
break
return losses.avg, errors.avg