def select_model(m):
if m == 'large':
model = pblm.mnist_model_large().cuda()
_, test_loader = pblm.mnist_loaders(8)
elif m == 'wide':
print("Using wide model with model_factor={}".format(args.model_factor))
_, test_loader = pblm.mnist_loaders(64//args.model_factor)
model = pblm.mnist_model_wide(args.model_factor).cuda()
elif m == 'deep':
print("Using deep model with model_factor={}".format(args.model_factor))
_, test_loader = pblm.mnist_loaders(64//(2**args.model_factor))
model = pblm.mnist_model_deep(args.model_factor).cuda()
else:
model = pblm.mnist_model().cuda()
return model
model = pblm.mnist_model_deep_custom().cuda()
elif m == '500':
model = pblm.mnist_500().cuda()
else:
model = pblm.mnist_model().cuda()
return model
if __name__ == "__main__":
args = pblm.argparser(opt='adam', verbose=200, starting_epsilon=0.01)
print("saving file to {}".format(args.prefix))
setproctitle.setproctitle(args.prefix)
train_log = open(args.prefix + "_train.log", "w")
test_log = open(args.prefix + "_test.log", "w")
train_loader, _ = pblm.mnist_loaders(args.batch_size)
_, test_loader = pblm.mnist_loaders(args.test_batch_size)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
for X, y in train_loader:
break
kwargs = pblm.args2kwargs(args, X=Variable(X.cuda()))
best_err = 1
sampler_indices = []
model = [select_model(args.model)]
for _ in range(0, args.cascade):
if _ > 0:
'Starting epsilon: {4}\t\t'
'Sechduled length: {5}'.format(args.type, args.category, args.epochs,
args.epsilon, args.starting_epsilon,
args.schedule_length),
end='\n')
if args.l1_proj is not None:
print('Projection vectors: {0}\t\t'
'Train estimation: {1}\t\t'
'Test estimation: {2}'.format(args.l1_proj, args.l1_train,
args.l1_test),
end='\n')
# train-validation split
train_loader, valid_loader, test_loader = pblm.mnist_loaders(
batch_size=args.batch_size,
path='../data',
ratio=args.ratio,
seed=args.seed)
model = pblm.mnist_model().cuda()
num_classes = model[-1].out_features
# specify the task and construct the corresponding cost matrix
folder_path = os.path.dirname(args.proctitle)
if args.type == 'binary':
input_mat = np.zeros((num_classes, num_classes), dtype=np.int)
if args.category == 'single_seed':
seed_clas = 9
input_mat[seed_clas, :] = np.ones(num_classes)
input_mat[seed_clas, seed_clas] = 0
folder_path += '/class_' + str(seed_clas)
else:
def plot_certificates():
# Load data
# The bounds in NN-space
x_min = 0.
x_max = 1.
# Fix random seed for reproducibility
seed = 0
np.random.seed(seed)
torch.manual_seed(seed)
_, test_loader = pblm.mnist_loaders(50)
# Get data into arrays for convenience
for idx, (data, target) in enumerate(test_loader):
if CUDA:
data, target = data.float().cuda(), target.long().cuda()
else:
data, target = data.float(), target.long()
#print(data.size())
#raise Exception()
data = data.view(-1, 1, 28, 28)
x = data
y = target
break
#print(x.size(), y.size())
#raise Exception()
#epochs = np.array([-1,0,2,4,6,9,14,19,24,32,41,49,61,74,86,99])
epochs = np.array([32, 41, 49, 61, 74, 86, 99])
robust_errs = []
for epoch in epochs:
epsilon = 0.1
model = pblm.mnist_model()
if epoch == -1:
model.load_state_dict(
torch.load(
'./snapshots/mnist_baseline_batch_size_50_epochs_100_lr_0.001_opt_adam_real_time_False_seed_0_checkpoint_99.pth'
))
else:
model.load_state_dict(
torch.load(
f'./snapshots/mnist_robustified_robust_batch_size_50_epochs_100_epsilon_0.1_l1_test_exact_l1_train_exact_lr_0.001_opt_adam_real_time_False_schedule_length_50_seed_0_starting_epsilon_0.01_checkpoint_{epoch}.pth'
))
if CUDA:
model.cuda()
_, robust_err = robust_loss(model, epsilon, x, y)
robust_errs.append(robust_err)
robust_errs = np.array(robust_errs)
results = pickle.load(
open(f'./snapshots/mnist_extracted_exp_results.pickle', 'rb'))
xs = np.array([-1, 0, 2, 4, 6, 9, 14, 19, 24, 32, 41, 49, 61, 74, 86, 99])
our_results = {}
for sigma in [0.1, 0.2, 0.3]:
sigma_lg_ps = []
our_results[sigma] = np.zeros(16)
for sample_id in range(50):
#print('sample id', sample_id)
#if sample_id == 24:
# print([r.shape for r in results[(sample_id,sigma)]])
# lg_ps = np.array(results[(sample_id,sigma)])
# print('lg_ps', lg_ps.shape)
#
# input()
lg_ps = np.array(results[(sample_id, sigma)])
#print('lg_ps', lg_ps.shape)
#input()
#print(lg_ps.shape)
#if len(lg_ps.shape) == 1:
# lg_ps = lg_ps.reshape((-1,1))
# #print(lg_ps.shape)
mean_ps = np.mean(lg_ps, axis=1)
#print(mean_ps.shape, lg_ps.shape)
our_results[sigma] += mean_ps != -250.0
#print(mean_ps)
#input()
#raise Exception()
#print(mean_ps.shape)
#raise Exception()
our_results[sigma] /= 50.0
fig = plt.figure(figsize=(cm2inch(8.0), cm2inch(6.0)))
ax = fig.add_subplot(1,
1,
1,
xlabel='epoch',
ylabel='fraction certified',
ylim=(-0.05, 1.0))
ax.plot(xs,
1. - our_results[0.3],
color='navy',
marker='.',
linewidth=1.0,
label=r'AMLS $\epsilon=0.3$')
ax.plot(xs,
1. - our_results[0.2],
color='seagreen',
marker='.',
linewidth=1.0,
label=r'AMLS $\epsilon=0.2$')
ax.plot(xs,
1. - our_results[0.1],
color='firebrick',
marker='.',
linewidth=1.0,
label=r'AMLS $\epsilon=0.1$')
ax.plot(epochs,
1. - robust_errs,
color='grey',
marker='.',
linestyle='--',
linewidth=1.0,
label=r'W\&K $\epsilon=0.1$')
#ax.legend(loc='lower right')
ax.legend(bbox_to_anchor=(0.9, 0.15),
loc="lower right",
bbox_transform=fig.transFigure)
ax.xaxis.set_tick_params(width=0.5)
ax.yaxis.set_tick_params(width=0.5)
ax.spines['left'].set_linewidth(0.5)
ax.spines['bottom'].set_linewidth(0.5)
sns.despine()
#fig.savefig(f'mnist_test_robust_losses.pdf', bbox_inches='tight')
fig.savefig(f'./results/robust/mnist_certificates.svg',
bbox_inches='tight')
plt.close(fig)
saved_filepath = ('./saved_log/' + args.proctitle)
model_filepath = os.path.dirname('./models/' + args.proctitle)
if not os.path.exists(saved_filepath):
os.makedirs(saved_filepath)
if not os.path.exists(model_filepath):
os.makedirs(model_filepath)
model_path = ('./models/' + args.proctitle)
train_res = open(saved_filepath + '/train_res.txt', "w")
test_res = open(saved_filepath + '/test_res.txt', "w")
# load the data
if args.prefix == "mnist":
train_loader, _ = pblm.mnist_loaders(
args.batch_size,
'../data/mnist',
)
_, test_loader = pblm.mnist_loaders(args.batch_size_test,
'../data/mnist')
elif args.prefix == "custom_mnist":
train_loader, _ = pblm.custom_mnist_loaders(
batch_size=args.batch_size,
train_path='../data/mnist/' + args.gan_type + '/train.npz',
test_path='../data/mnist/' + args.gan_type + '/test.npz')
_, test_loader = pblm.custom_mnist_loaders(
batch_size=args.batch_size_test,
train_path='../data/mnist/' + args.gan_type + '/train.npz',
test_path='../data/mnist/' + args.gan_type + '/test.npz')
else:
