def train_robust(loader, model, opt, epsilon, epoch, log):
model.train()
if epoch == 0:
blank_state = opt.state_dict()
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda()
robust_ce, robust_err = robust_loss(model, epsilon, Variable(X),
Variable(y))
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.data.max(1)[1] != y).float().sum() / X.size(0)
opt.zero_grad()
robust_ce.backward()
opt.step()
print(epoch,
i,
robust_ce.data[0],
robust_err[0],
ce.data[0],
err,
file=log)
print(epoch, i, robust_ce.data[0], robust_err[0], ce.data[0], err)
log.flush()
def evaluate_robust(loader, model, epsilon, epoch, log, verbose):
model.eval()
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda().long()
if y.dim() == 2:
y = y.squeeze(1)
robust_ce, robust_err = robust_loss(model,
epsilon,
Variable(X, volatile=True),
Variable(y, volatile=True),
alpha_grad=True,
scatter_grad=True)
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.data.max(1)[1] != y).float().sum() / X.size(0)
print(epoch,
i,
robust_ce.data[0],
robust_err,
ce.data[0],
err,
file=log)
if i % verbose == 0:
print(epoch, i, robust_ce.data[0], robust_err, ce.data[0], err)
log.flush()
del X, y, robust_ce, out, ce
def evaluate_robust(loader, model, epsilon, epoch, log):
model.eval()
for i, (X,y) in enumerate(loader):
X,y = X.cuda(), y.cuda()
robust_ce, robust_err = robust_loss(model, epsilon,
Variable(X), Variable(y))
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.data.max(1)[1] != y).float().sum() / X.size(0)
print(epoch, i, robust_ce.data[0], robust_err[0], ce.data[0], err, log)
print(epoch, i, robust_ce.data[0], robust_err[0], ce.data[0], err)
log.flush()
def evaluate_robust(loader, model, epsilon, log, verbose):
batch_time = AverageMeter()
robust_losses = AverageMeter()
robust_errors = AverageMeter()
end = time.time()
torch.set_grad_enabled(False)
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda().long()
if y.dim() == 2:
y = y.squeeze(1)
robust_ce, robust_err = robust_loss(model, epsilon, X, y)
# measure accuracy and record loss
robust_losses.update(robust_ce.item(), X.size(0))
robust_errors.update(robust_err, X.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# print(i, robust_ce.item(), robust_err, file=log)
if verbose:
endline = '\n' if i % verbose == 0 else '\r'
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Robust loss {rloss.val:.3f} ({rloss.avg:.3f})\t'
'Robust error {rerrors.val:.3f} ({rerrors.avg:.3f})'.format(
i,
len(loader),
batch_time=batch_time,
rloss=robust_losses,
rerrors=robust_errors),
end=endline)
log.flush()
del X, y, robust_ce
if DEBUG and i == 10:
break
torch.set_grad_enabled(True)
torch.cuda.empty_cache()
return robust_losses.avg, robust_errors.avg
def train_robust(loader, model, opt, epsilon, epoch, log, verbose, alpha_grad,
scatter_grad, l1_proj):
model.train()
if epoch == 0:
blank_state = opt.state_dict()
for i, (X, y) in enumerate(loader):
X, y = X.cuda(), y.cuda().long()
if y.dim() == 2:
y = y.squeeze(1)
robust_ce, robust_err = robust_loss(model,
epsilon,
Variable(X),
Variable(y),
alpha_grad=alpha_grad,
scatter_grad=scatter_grad)
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.data.max(1)[1] != y).float().sum() / X.size(0)
opt.zero_grad()
robust_ce.backward()
opt.step()
print(epoch,
i,
robust_ce.data[0],
robust_err,
ce.data[0],
err,
file=log)
if i % verbose == 0:
print(epoch, i, robust_ce.data[0], robust_err, ce.data[0], err)
log.flush()
del X, y, robust_ce, out, ce
def attack(loader, model, epsilon, verbose=False, atk=None, robust=False):
total_err, total_fgs, total_robust = [], [], []
if verbose:
print("Requiring no gradients for parameters.")
for p in model.parameters():
p.requires_grad = False
for i, (X, y) in enumerate(loader):
X, y = Variable(X.cuda(),
requires_grad=True), Variable(y.cuda().long())
if y.dim() == 2:
y = y.squeeze(1)
if robust:
robust_ce, robust_err = robust_loss(model, epsilon, X, y, False,
False)
err, err_fgs = atk(model, X, y, epsilon)
total_err.append(err)
total_fgs.append(err_fgs)
if robust:
total_robust.append(robust_err)
if verbose:
if robust:
print('err: {} | attack: {} | robust: {}'.format(
err, err_fgs, robust_err))
else:
print('err: {} | attack: {}'.format(err, err_fgs))
if robust:
print('[TOTAL] err: {} | attack: {} | robust: {}'.format(
mean(total_err), mean(total_fgs), mean(total_robust)))
else:
print('[TOTAL] err: {} | attack: {}'.format(mean(total_err),
mean(total_fgs)))
return total_err, total_fgs, total_robust
def evaluate_robust(loader, model, epsilon, epoch, log, verbose,
real_time=False, parallel=False, **kwargs):
batch_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
robust_losses = AverageMeter()
robust_errors = AverageMeter()
model.eval()
end = time.time()
torch.set_grad_enabled(False)
for i, (X,y) in enumerate(loader):
X,y = X.cuda(), y.cuda().long()
if y.dim() == 2:
y = y.squeeze(1)
robust_ce, robust_err = robust_loss(model, epsilon, X, y, **kwargs)
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.max(1)[1] != y).float().sum() / X.size(0)
# _,pgd_err = _pgd(model, Variable(X), Variable(y), epsilon)
# measure accuracy and record loss
losses.update(ce.item(), X.size(0))
errors.update(err, X.size(0))
robust_losses.update(robust_ce.item(), X.size(0))
robust_errors.update(robust_err, X.size(0))
# measure elapsed time
batch_time.update(time.time()-end)
end = time.time()
print(epoch, i, robust_ce.item(), robust_err, ce.item(), err.item(),
file=log)
if verbose:
# print(epoch, i, robust_ce.data[0], robust_err, ce.data[0], err)
endline = '\n' if i % verbose == 0 else '\r'
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Robust loss {rloss.val:.3f} ({rloss.avg:.3f})\t'
'Robust error {rerrors.val:.3f} ({rerrors.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {error.val:.3f} ({error.avg:.3f})'.format(
i, len(loader), batch_time=batch_time,
loss=losses, error=errors, rloss = robust_losses,
rerrors = robust_errors), end=endline)
log.flush()
del X, y, robust_ce, out, ce
if DEBUG and i ==10:
break
torch.set_grad_enabled(True)
torch.cuda.empty_cache()
print('')
print(' * Robust error {rerror.avg:.3f}\t'
'Error {error.avg:.3f}'
.format(rerror=robust_errors, error=errors))
return robust_errors.avg
def train_robust(loader, model, opt, epsilon, epoch, log, verbose,
real_time=False, clip_grad=None, **kwargs):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
robust_losses = AverageMeter()
robust_errors = AverageMeter()
model.train()
end = time.time()
for i, (X,y) in enumerate(loader):
X,y = X.cuda(), y.cuda().long()
if y.dim() == 2:
y = y.squeeze(1)
data_time.update(time.time() - end)
with torch.no_grad():
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.max(1)[1] != y).float().sum() / X.size(0)
robust_ce, robust_err = robust_loss(model, epsilon,
Variable(X), Variable(y),
**kwargs)
opt.zero_grad()
robust_ce.backward()
if clip_grad:
nn.utils.clip_grad_norm_(model.parameters(), clip_grad)
opt.step()
# measure accuracy and record loss
losses.update(ce.item(), X.size(0))
errors.update(err.item(), X.size(0))
robust_losses.update(robust_ce.detach().item(), X.size(0))
robust_errors.update(robust_err, X.size(0))
# measure elapsed time
batch_time.update(time.time()-end)
end = time.time()
print(epoch, i, robust_ce.detach().item(),
robust_err, ce.item(), err.item(), file=log)
if verbose and (i % verbose == 0 or real_time):
endline = '\n' if i % verbose == 0 else '\r'
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Robust loss {rloss.val:.4f} ({rloss.avg:.4f})\t'
'Robust error {rerrors.val:.3f} ({rerrors.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {errors.val:.3f} ({errors.avg:.3f})'.format(
epoch, i, len(loader), batch_time=batch_time,
data_time=data_time, loss=losses, errors=errors,
rloss = robust_losses, rerrors = robust_errors), end=endline)
log.flush()
del X, y, robust_ce, out, ce, err, robust_err
if DEBUG and i ==10:
break
print('')
torch.cuda.empty_cache()
def evaluate_ensemble(model_paths):
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
test_loader = get_mnist_test_loader()
models = read_models(model_paths)
avg_model = averaging_model(models)
models.append(avg_model)
rows = list(csv.reader(open('examples/random_indices.csv')))
idx = set([int(row[0]) for row in rows[1:1001]])
header = [
'eps', 'unanm_norm_err', 'unanm_norm_rej', 'unanm_rob_certs',
'maj_norm_err', 'maj_norm_rej', 'maj_rob_certs', 'avg_norm_err'
]
header += [
'model_' + str(i + 1) + '_rob_certs' for i in range(len(models) - 1)
]
print(','.join([col for col in header]))
for k in range(1, 21):
unanm = {'norm_errs': 0, 'norm_rejs': 0, 'rob_certs': 0}
maj = {'norm_errs': 0, 'norm_rejs': 0, 'rob_certs': 0}
avg_norm_errs = 0
indiv_rob_certs = [0 for _ in range(len(models) - 1)]
eps = k / 100.0
for i, (X, y) in enumerate(test_loader):
if i not in idx:
continue
X, y = X.cuda(), y.cuda()
true_class = y.data.item()
preds = [model(X).data for model in models[:-1]]
pred_classes = [pred.max(1)[1].item() for pred in preds]
counter = Counter(pred_classes)
max_class = -1
max_count = 0
for c in counter:
if counter[c] > max_count:
max_count = counter[c]
max_class = c
if max_count == len(preds) and max_class == true_class:
pass
elif max_count == len(preds) and max_class != true_class:
unanm['norm_errs'] += 1
else:
unanm['norm_rejs'] += 1
if max_count >= len(preds) // 2 + 1 and max_class == true_class:
pass
elif max_count >= len(preds) // 2 + 1 and max_class != true_class:
maj['norm_errs'] += 1
else:
maj['norm_rejs'] += 1
preds_avg = models[-1](X).data
if preds_avg.max(1)[1].item() == true_class:
pass
else:
avg_norm_errs += 1
robust_errs = []
for i, model in enumerate(models):
_, robust_err = robust_loss(model,
eps,
X,
y,
bounded_input=False)
robust_errs.append(int(robust_err))
if sum(robust_errs) < len(robust_errs):
unanm['rob_certs'] += 1
elif sum(robust_errs[:-1]) < len(robust_errs[:-1]) // 2 + 1:
maj['rob_certs'] += 1
for i in range(len(indiv_rob_certs)):
indiv_rob_certs[i] += (1 - robust_errs[i])
lst = [(k / 100.0), unanm['norm_errs'], unanm['norm_rejs'],
unanm['rob_certs'], maj['norm_errs'], maj['norm_rejs'],
maj['rob_certs'], avg_norm_errs]
lst += indiv_rob_certs
print(','.join([str(t) for t in lst]))
print("train")
for epoch in range(nbepoch):
print("epoch=", epoch, "/", nbepoch)
net.train()
total, correct = 0, 0
for _, (inputs, targets) in enumerate(trainloader):
inputs, targets = inputs.to(device), targets.to(device)
if debug or epoch == 0:
outputs = net(inputs)
loss = criterion(outputs, targets)
else:
with torch.no_grad():
outputs = net(inputs)
loss, _ = convex_adversarial.robust_loss(net, 3.0 / 255, inputs, targets)
meanloss.append(loss.cpu().data.numpy())
if epoch > 0:
loss *= 0.0001
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(net.parameters(), 1)
optimizer.step()
_, predicted = outputs.max(1)
total += targets.size(0)
correct += predicted.eq(targets).sum().item()
def robust_loss_cascade(models, epsilon, X, y, **kwargs):
total_robust_ce = 0.
total_ce = 0.
total_robust_err = 0.
total_err = 0.
batch_size = float(X.size(0))
I = torch.arange(X.size(0)).type_as(y.data)
for j, model in enumerate(models[:-1]):
out = model(X)
ce = nn.CrossEntropyLoss(reduce=False)(out, y)
_, uncertified = robust_loss(model,
epsilon,
X,
out.max(1)[1],
size_average=False,
**kwargs)
certified = ~uncertified
l = []
if certified.sum() == 0:
pass
# print("Warning: Cascade stage {} has no certified values.".format(j+1))
else:
X_cert = X[Variable(certified.nonzero()[:, 0])]
y_cert = y[Variable(certified.nonzero()[:, 0])]
ce = ce[Variable(certified.nonzero()[:, 0])]
out = out[Variable(certified.nonzero()[:, 0])]
err = (out.data.max(1)[1] != y_cert.data).float()
robust_ce, robust_err = robust_loss(model,
epsilon,
X_cert,
y_cert,
size_average=False,
**kwargs)
# add statistics for certified examples
total_robust_ce += robust_ce.sum()
total_ce += ce.data.sum()
total_robust_err += robust_err.sum()
total_err += err.sum()
l.append(certified.sum())
# reduce data set to uncertified examples
if uncertified.sum() > 0:
X = X[Variable(uncertified.nonzero()[:, 0])]
y = y[Variable(uncertified.nonzero()[:, 0])]
I = I[uncertified.nonzero()[:, 0]]
else:
robust_ce = total_robust_ce / batch_size
ce = total_ce / batch_size
robust_err = total_robust_err.item() / batch_size
err = total_err.item() / batch_size
return robust_ce, robust_err, ce, err, None
####################################################################
# compute normal ce and robust ce for the last model
out = models[-1](X)
ce = nn.CrossEntropyLoss(reduce=False)(out, y)
err = (out.data.max(1)[1] != y.data).float()
robust_ce, robust_err = robust_loss(models[-1],
epsilon,
X,
y,
size_average=False,
**kwargs)
# update statistics with the remaining model and take the average
total_robust_ce += robust_ce.sum()
total_ce += ce.data.sum()
total_robust_err += robust_err.sum()
total_err += err.sum()
robust_ce = total_robust_ce / batch_size
ce = total_ce / batch_size
robust_err = total_robust_err.item() / batch_size
err = total_err.item() / batch_size
_, uncertified = robust_loss(models[-1],
epsilon,
X,
out.max(1)[1],
size_average=False,
**kwargs)
if uncertified.sum() > 0:
I = I[uncertified.nonzero()[:, 0]]
else:
I = None
return robust_ce, robust_err, ce, err, I
# torch.manual_seed(1)
robust_net = nn.Sequential(nn.Conv2d(1, j, 3, stride=1, padding=1),
nn.ReLU(), Flatten(),
nn.Linear(j * 28 * 28, 2)).cuda()
data = []
opt = optim.Adam(robust_net.parameters(), lr=1e-3)
ts = []
for i in range(10):
start_time = time.time()
if PROJ:
robust_ce, robust_err = robust_loss(robust_net,
epsilon,
X,
y,
parallel=False,
l1_proj=50,
l1_type='median')
else:
robust_ce, robust_err = robust_loss(robust_net,
epsilon,
X,
y,
parallel=False)
out = robust_net(X)
l2 = nn.CrossEntropyLoss()(out, y).item()
err = (out.max(1)[1] != y).float().mean().item()
data.append([l2, robust_ce.item(), err, robust_err])
# if i % 100 == 0:
if min(np.max(np.abs(p - a)) for a in x) > 3 * r:
x.append(p)
X = tr.tensor(np.array(x), dtype=tr.float32)
tr.manual_seed(1)
y = (tr.rand(m) + 0.5).long()
# Inizializzaimo una rete neurale
tr.manual_seed(1)
net = nn.Sequential(nn.Linear(2, 100), nn.ReLU(), nn.Linear(100, 100),
nn.ReLU(), nn.Linear(100, 100), nn.ReLU(),
nn.Linear(100, 100), nn.ReLU(), nn.Linear(100, 2))
epoch = 1000
learning_rate = 1e-3
opt = optim.Adam(net.parameters(), lr=learning_rate)
for i in range(epoch):
loss, err = robust_loss(net, r, X, y)
print('Epoch : %d, error : %e' % (i + 1, loss.item()), end='\r')
# err = (out.max(1)[1].item != y).float().mean()
opt.zero_grad()
loss.backward()
opt.step()
print('Error : ' + str(loss.item()))
print('Training :' + str(x))
print('Targets :' + str(y))
l = 1
for par in net.parameters():
if len(par.data.shape) > 1:
l = l * par.data.norm(1, 1).max()
print('Lipschitz norm of the network : ' + str(l.item()))
eps_max = 0.05
print('Optimization method : ')
print(opt)
epochs = 4
print('Start Learning Process...')
for epoch in range(epochs):
total_loss = 0.0
train_iter = iter(trainloader)
for i, data in enumerate(trainloader, 0):
inputs, labels = data
# Training relu network
if epoch < 3:
out = net(inputs)
loss = lossy_fn(out, labels)
else:
loss, err = robust_loss(net, eps_max, inputs, labels)
opt.zero_grad()
loss.backward()
opt.step()
if i % 20 == 19:
print(' Epoch : %d, training object : %5d, local error : %.2e '
% (epoch + 1, (i + 1)*batch_size, loss.item()), end='\r')
total_loss += loss.item()
print()
print(' Epoch : %d, main error : %e' % (epoch + 1, total_loss/((i+1)*batch_size)))
print('End of Training.')
print(' ')
print('Start of Testing...')
eps_set = (0.001*(1+np.arange(9))).tolist() + (0.01*(1+np.arange(10))).tolist()
corr_unit = list(0. for i in range(len(eps_set)))
def train_robust(loader, model, opt, epsilon, epoch, log, verbose,
real_time=False, clip_grad=None, **kwargs):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
errors = AverageMeter()
robust_losses = AverageMeter()
robust_errors = AverageMeter()
model.train()
end = time.time()
NAN_EXIST = False
NAN_count = 0
for i, (X,y) in enumerate(loader):
X,y = X.cuda(), y.cuda().long()
if y.dim() == 2:
y = y.squeeze(1)
data_time.update(time.time() - end)
with torch.no_grad():
out = model(Variable(X))
ce = nn.CrossEntropyLoss()(out, Variable(y))
err = (out.max(1)[1] != y).float().sum() / X.size(0)
robust_ce, robust_err = robust_loss(model, epsilon,
Variable(X), Variable(y),
**kwargs)
###############################################################
if np.isnan(ce.item()):
print('Natural Loss goes to NAN VALUE!!!')
if np.isnan(robust_ce.detach().item()):
print('Adversarial Loss goes to NAN VALUE!!!')
opt.zero_grad()
robust_ce.backward()
############################################################
for p in model.parameters():
#nan_idx = np.argwhere(np.isnan(p.grad))
if torch.sum(torch.isnan(p.grad))>0:
if NAN_EXIST == False:
#
NAN_EXIST = True
if NAN_count == 0:
print('!!!!!!!!!!!!!!!!!!!!!!!!! Number of NAN elements: ', torch.sum(torch.isnan(p.grad)))
#break
p.grad[torch.isnan(p.grad)] = 0
#print(torch.sum(p.grad>10))
#nan_idx = np.argwhere(p.grad<=100)
#print(p.grad.shape)
#print(nan_idx.shape)
#p.grad[nan_idx] = 0
'''
if NAN_EXIST == False:
for p in model.parameters():
if np.isnan(p.grad).any():
print('Gradient just goes to NAN VALUE!!!!!!!!!!!!!!!!!!!!!!!!!!')
NAN_EXIST = True
break
if NAN_EXIST == True:
break
'''
############################################################
if NAN_EXIST == False:
if clip_grad:
nn.utils.clip_grad_norm_(model.parameters(), clip_grad)
opt.step()
else:
opt.step()
NAN_count += 1
NAN_EXIST = False
# measure accuracy and record loss
losses.update(ce.item(), X.size(0))
errors.update(err.item(), X.size(0))
robust_losses.update(robust_ce.detach().item(), X.size(0))
robust_errors.update(robust_err, X.size(0))
# measure elapsed time
batch_time.update(time.time()-end)
end = time.time()
print(epoch, i, robust_ce.detach().item(),
robust_err, ce.item(), err.item(), file=log)
if verbose and (i % verbose == 0 or real_time):
endline = '\n' if i % verbose == 0 else '\r'
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Robust loss {rloss.val:.4f} ({rloss.avg:.4f})\t'
'Robust error {rerrors.val:.3f} ({rerrors.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Error {errors.val:.3f} ({errors.avg:.3f})'.format(
epoch, i, len(loader), batch_time=batch_time,
data_time=data_time, loss=losses, errors=errors,
rloss = robust_losses, rerrors = robust_errors), end=endline)
log.flush()
del X, y, robust_ce, out, ce, err, robust_err
if DEBUG and i ==10:
break
#if i==100:
# break
print('In totoal, the number of NAN gradient elements happen is : ', NAN_count)
print('')
torch.cuda.empty_cache()
break
if args.method == "convexdual" or args.compare_all:
from convex_adversarial import robust_loss
'''
if(PARALLEL):
# The first run with parallel will take a few seconds
# to warm up.
eric_loss, eric_err = robust_loss(model,\
epsilon, X, y, parallel=PARALLEL)
del eric_loss, eric_err
'''
start = time.time()
eric_loss, eric_err = robust_loss(model,\
epsilon, X, y,parallel=True,\
bounded_input={0, 1})
#norm_type="l1_median", proj=20)
#eric_loss, eric_err = robust_loss(model,\
#epsilon, X, y, parallel=PARALLEL)
print("eric loss", eric_loss)
print("eric err:", eric_err)
print("eric time per sample:", (time.time() - start) / X.shape[0])
del eric_loss, eric_err
print()
if args.method == "baseline" or args.compare_all:
#if(method == BASELINE):
start = time.time()
def plot_certificates():
# Load data
# The bounds in NN-space
x_min = 0.
x_max = 1.
# Fix random seed for reproducibility
seed = 0
np.random.seed(seed)
torch.manual_seed(seed)
_, test_loader = pblm.mnist_loaders(50)
# Get data into arrays for convenience
for idx, (data, target) in enumerate(test_loader):
if CUDA:
data, target = data.float().cuda(), target.long().cuda()
else:
data, target = data.float(), target.long()
#print(data.size())
#raise Exception()
data = data.view(-1, 1, 28, 28)
x = data
y = target
break
#print(x.size(), y.size())
#raise Exception()
#epochs = np.array([-1,0,2,4,6,9,14,19,24,32,41,49,61,74,86,99])
epochs = np.array([32, 41, 49, 61, 74, 86, 99])
robust_errs = []
for epoch in epochs:
epsilon = 0.1
model = pblm.mnist_model()
if epoch == -1:
model.load_state_dict(
torch.load(
'./snapshots/mnist_baseline_batch_size_50_epochs_100_lr_0.001_opt_adam_real_time_False_seed_0_checkpoint_99.pth'
))
else:
model.load_state_dict(
torch.load(
f'./snapshots/mnist_robustified_robust_batch_size_50_epochs_100_epsilon_0.1_l1_test_exact_l1_train_exact_lr_0.001_opt_adam_real_time_False_schedule_length_50_seed_0_starting_epsilon_0.01_checkpoint_{epoch}.pth'
))
if CUDA:
model.cuda()
_, robust_err = robust_loss(model, epsilon, x, y)
robust_errs.append(robust_err)
robust_errs = np.array(robust_errs)
results = pickle.load(
open(f'./snapshots/mnist_extracted_exp_results.pickle', 'rb'))
xs = np.array([-1, 0, 2, 4, 6, 9, 14, 19, 24, 32, 41, 49, 61, 74, 86, 99])
our_results = {}
for sigma in [0.1, 0.2, 0.3]:
sigma_lg_ps = []
our_results[sigma] = np.zeros(16)
for sample_id in range(50):
#print('sample id', sample_id)
#if sample_id == 24:
# print([r.shape for r in results[(sample_id,sigma)]])
# lg_ps = np.array(results[(sample_id,sigma)])
# print('lg_ps', lg_ps.shape)
#
# input()
lg_ps = np.array(results[(sample_id, sigma)])
#print('lg_ps', lg_ps.shape)
#input()
#print(lg_ps.shape)
#if len(lg_ps.shape) == 1:
# lg_ps = lg_ps.reshape((-1,1))
# #print(lg_ps.shape)
mean_ps = np.mean(lg_ps, axis=1)
#print(mean_ps.shape, lg_ps.shape)
our_results[sigma] += mean_ps != -250.0
#print(mean_ps)
#input()
#raise Exception()
#print(mean_ps.shape)
#raise Exception()
our_results[sigma] /= 50.0
fig = plt.figure(figsize=(cm2inch(8.0), cm2inch(6.0)))
ax = fig.add_subplot(1,
1,
1,
xlabel='epoch',
ylabel='fraction certified',
ylim=(-0.05, 1.0))
ax.plot(xs,
1. - our_results[0.3],
color='navy',
marker='.',
linewidth=1.0,
label=r'AMLS $\epsilon=0.3$')
ax.plot(xs,
1. - our_results[0.2],
color='seagreen',
marker='.',
linewidth=1.0,
label=r'AMLS $\epsilon=0.2$')
ax.plot(xs,
1. - our_results[0.1],
color='firebrick',
marker='.',
linewidth=1.0,
label=r'AMLS $\epsilon=0.1$')
ax.plot(epochs,
1. - robust_errs,
color='grey',
marker='.',
linestyle='--',
linewidth=1.0,
label=r'W\&K $\epsilon=0.1$')
#ax.legend(loc='lower right')
ax.legend(bbox_to_anchor=(0.9, 0.15),
loc="lower right",
bbox_transform=fig.transFigure)
ax.xaxis.set_tick_params(width=0.5)
ax.yaxis.set_tick_params(width=0.5)
ax.spines['left'].set_linewidth(0.5)
ax.spines['bottom'].set_linewidth(0.5)
sns.despine()
#fig.savefig(f'mnist_test_robust_losses.pdf', bbox_inches='tight')
fig.savefig(f'./results/robust/mnist_certificates.svg',
bbox_inches='tight')
plt.close(fig)
