def main(net, spec, verbose=True):
with open(spec, 'r') as f:
lines = [line[:-1] for line in f.readlines()]
true_label = int(lines[0])
pixel_values = [float(line) for line in lines[1:]]
eps = float(spec[:-4].split('/')[-1].split('_')[-1])
if net == 'fc1':
nn = FullyConnected(DEVICE, INPUT_SIZE, [50, 10]).to(DEVICE)
elif net == 'fc2':
nn = FullyConnected(DEVICE, INPUT_SIZE, [100, 50, 10]).to(DEVICE)
elif net == 'fc3':
nn = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
elif net == 'fc4':
nn = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 50, 10]).to(DEVICE)
elif net == 'fc5':
nn = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 100, 10]).to(DEVICE)
elif net == 'fc6':
nn = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 100, 10]).to(DEVICE)
elif net == 'fc7':
nn = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 100, 100, 10]).to(DEVICE)
elif net == 'conv1':
nn = Conv(DEVICE, INPUT_SIZE, [(16, 3, 2, 1)], [100, 10],
10).to(DEVICE)
elif net == 'conv2':
nn = Conv(DEVICE, INPUT_SIZE, [(16, 4, 2, 1), (32, 4, 2, 1)],
[100, 10], 10).to(DEVICE)
elif net == 'conv3':
nn = Conv(DEVICE, INPUT_SIZE, [(16, 4, 2, 1), (64, 4, 2, 1)],
[100, 100, 10], 10).to(DEVICE)
else:
assert False
nn.load_state_dict(
torch.load('../mnist_nets/%s.pt' % net,
map_location=torch.device(DEVICE)))
inputs = torch.FloatTensor(pixel_values).view(1, 1, INPUT_SIZE,
INPUT_SIZE).to(DEVICE)
outs = nn(inputs)
pred_label = outs.max(dim=1)[1].item()
assert pred_label == true_label
res = analyze(net, nn, inputs, eps, true_label)
out = "verified" if res else 'not verified'
if verbose:
print(out)
return out
def main():
parser = argparse.ArgumentParser(description='Neural network verification using DeepZ relaxation')
parser.add_argument('--net',
type=str,
choices=['fc1', 'fc2', 'fc3', 'fc4', 'fc5', 'conv1', 'conv2', 'conv3', 'conv4', 'conv5'],
required=True,
help='Neural network to verify.')
parser.add_argument('--spec', type=str, required=True, help='Test case to verify.')
args = parser.parse_args()
with open(args.spec, 'r') as f:
lines = [line[:-1] for line in f.readlines()]
true_label = int(lines[0])
pixel_values = [float(line) for line in lines[1:]]
eps = float(args.spec[:-4].split('/')[-1].split('_')[-1])
if args.net == 'fc1':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 10]).to(DEVICE)
elif args.net == 'fc2':
net = FullyConnected(DEVICE, INPUT_SIZE, [50, 50, 10]).to(DEVICE)
elif args.net == 'fc3':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
elif args.net == 'fc4':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 100, 10]).to(DEVICE)
elif args.net == 'fc5':
net = FullyConnected(DEVICE, INPUT_SIZE, [400, 200, 100, 100, 10]).to(DEVICE)
elif args.net == 'conv1':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1)], [100, 10], 10).to(DEVICE)
elif args.net == 'conv2':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)], [100, 10], 10).to(DEVICE)
elif args.net == 'conv3':
net = Conv(DEVICE, INPUT_SIZE, [(32, 3, 1, 1), (32, 4, 2, 1), (64, 4, 2, 1)], [150, 10], 10).to(DEVICE)
elif args.net == 'conv4':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)], [100, 100, 10], 10).to(DEVICE)
elif args.net == 'conv5':
net = Conv(DEVICE, INPUT_SIZE, [(16, 3, 1, 1), (32, 4, 2, 1), (64, 4, 2, 1)], [100, 100, 10], 10).to(DEVICE)
net.load_state_dict(torch.load('../mnist_nets/%s.pt' % args.net, map_location=torch.device(DEVICE)))
inputs = torch.FloatTensor(pixel_values).view(1, 1, INPUT_SIZE, INPUT_SIZE).to(DEVICE)
outs = net(inputs)
pred_label = outs.max(dim=1)[1].item()
assert pred_label == true_label
if analyze(net, inputs, eps, true_label):
print('verified')
else:
print('not verified')
except:
adv_ex_found = pd.DataFrame(np.zeros(shape=(len(choices), eps_steps)),
columns=eps_space,
index=choices)
print("Created new overview")
else:
adv_ex_found = pd.DataFrame(np.zeros(shape=(len(choices), eps_steps)),
columns=eps_space,
index=choices)
for i_net, net in enumerate(choices):
if i_net < skip_nets:
continue
strnet = net
if net == 'fc1':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 10]).to(DEVICE)
elif net == 'fc2':
net = FullyConnected(DEVICE, INPUT_SIZE, [50, 50, 10]).to(DEVICE)
elif net == 'fc3':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
elif net == 'fc4':
net = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 10]).to(DEVICE)
elif net == 'fc5':
net = FullyConnected(DEVICE, INPUT_SIZE,
[400, 200, 100, 100, 10]).to(DEVICE)
elif net == 'conv1':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1)], [100, 10],
10).to(DEVICE)
elif net == 'conv2':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)],
def main():
with open(args.spec, "r") as f:
lines = [line[:-1] for line in f.readlines()]
true_label = int(lines[0])
pixel_values = [float(line) for line in lines[1:]]
eps = float(args.spec[:-4].split("/")[-1].split("_")[-1])
if args.net == "fc1":
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 10]).to(DEVICE)
elif args.net == "fc2":
net = FullyConnected(DEVICE, INPUT_SIZE, [50, 50, 10]).to(DEVICE)
elif args.net == "fc3":
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
elif args.net == "fc4":
net = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 10]).to(DEVICE)
elif args.net == "fc5":
net = FullyConnected(DEVICE, INPUT_SIZE,
[400, 200, 100, 100, 10]).to(DEVICE)
elif args.net == "conv1":
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1)], [100, 10],
10).to(DEVICE)
elif args.net == "conv2":
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)],
[100, 10], 10).to(DEVICE)
elif args.net == "conv3":
net = Conv(DEVICE, INPUT_SIZE, [(32, 3, 1, 1), (32, 4, 2, 1),
(64, 4, 2, 1)], [150, 10],
10).to(DEVICE)
elif args.net == "conv4":
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)],
[100, 100, 10], 10).to(DEVICE)
elif args.net == "conv5":
net = Conv(DEVICE, INPUT_SIZE, [(16, 3, 1, 1), (32, 4, 2, 1),
(64, 4, 2, 1)], [100, 100, 10],
10).to(DEVICE)
net.load_state_dict(
torch.load("../mnist_nets/%s.pt" % args.net,
map_location=torch.device(DEVICE)))
inputs = torch.FloatTensor(pixel_values).view(1, 1, INPUT_SIZE,
INPUT_SIZE).to(DEVICE)
outs = net(inputs)
pred_label = outs.max(dim=1)[1].item()
assert pred_label == true_label
if analyze(net, inputs, eps, true_label):
print("verified")
else:
print("not verified")
def main():
parser = argparse.ArgumentParser(description='Neural network verification using DeepZ relaxation')
parser.add_argument('--net',
type=str,
choices=['fc1', 'fc2', 'fc3', 'fc4', 'fc5', 'conv1', 'conv2', 'conv3', 'conv4', 'conv5'],
required=True,
help='Neural network to verify.')
args = parser.parse_args()
if args.net == 'fc1':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 10]).to(DEVICE)
elif args.net == 'fc2':
net = FullyConnected(DEVICE, INPUT_SIZE, [50, 50, 10]).to(DEVICE)
elif args.net == 'fc3':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
elif args.net == 'fc4':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 100, 10]).to(DEVICE)
elif args.net == 'fc5':
net = FullyConnected(DEVICE, INPUT_SIZE, [400, 200, 100, 100, 10]).to(DEVICE)
elif args.net == 'conv1':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1)], [100, 10], 10).to(DEVICE)
elif args.net == 'conv2':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)], [100, 10], 10).to(DEVICE)
elif args.net == 'conv3':
net = Conv(DEVICE, INPUT_SIZE, [(32, 3, 1, 1), (32, 4, 2, 1), (64, 4, 2, 1)], [150, 10], 10).to(DEVICE)
elif args.net == 'conv4':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)], [100, 100, 10], 10).to(DEVICE)
elif args.net == 'conv5':
net = Conv(DEVICE, INPUT_SIZE, [(16, 3, 1, 1), (32, 4, 2, 1), (64, 4, 2, 1)], [100, 100, 10], 10).to(DEVICE)
net.load_state_dict(torch.load('../mnist_nets/%s.pt' % args.net, map_location=torch.device(DEVICE)))
test_loader = torch.utils.data.DataLoader(
torchvision.datasets.MNIST('../data/', train=False, download=True,
transform=torchvision.transforms.Compose([
torchvision.transforms.ToTensor()
])),
batch_size=1, shuffle=True)
examples = enumerate(test_loader)
eps = 0.02
fnet = LinfPGDAttack(net, epsilon=eps, k=40)
point = 0
for batch_idx, (x, y) in examples:
X_adv = fnet.perturb(x.numpy(), y.numpy())
X = torch.from_numpy(X_adv)
if net(X).max(dim=1)[1].item() == y.item():
out = 'verified'
else:
out = 'not verified'
print(out)
if analyze(net, X, eps, y.item()):
pred = 'verified'
else:
pred = 'not verified'
print(pred)
print('-----------')
if out == pred:
point += 1
if out == 'not verified' and pred == 'verified':
point -= 2
print('marks ', point)
def core_analysis(net_str,true_label,pixel_values,eps,VERBOSE=0):
start=time.time()
if net_str == 'fc1':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 10]).to(DEVICE)
elif net_str == 'fc2':
net = FullyConnected(DEVICE, INPUT_SIZE, [50, 50, 10]).to(DEVICE)
elif net_str == 'fc3':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
elif net_str == 'fc4':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 100, 10]).to(DEVICE)
elif net_str == 'fc5':
net = FullyConnected(DEVICE, INPUT_SIZE, [400, 200, 100, 100, 10]).to(DEVICE)
elif net_str == 'conv1':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1)], [100, 10], 10).to(DEVICE)
elif net_str == 'conv2':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)], [100, 10], 10).to(DEVICE)
elif net_str == 'conv3':
net = Conv(DEVICE, INPUT_SIZE, [(32, 3, 1, 1), (32, 4, 2, 1), (64, 4, 2, 1)], [150, 10], 10).to(DEVICE)
elif net_str == 'conv4':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)], [100, 100, 10], 10).to(DEVICE)
elif net_str == 'conv5':
net = Conv(DEVICE, INPUT_SIZE, [(16, 3, 1, 1), (32, 4, 2, 1), (64, 4, 2, 1)], [100, 100, 10], 10).to(DEVICE)
net.load_state_dict(torch.load(os.path.join(os.path.dirname(__file__),'../mnist_nets/%s.pt' % net_str), map_location=torch.device(DEVICE)))
inputs = torch.FloatTensor(np.array(pixel_values).reshape((1, 1, INPUT_SIZE, INPUT_SIZE))).to(DEVICE)
outs = net(inputs)
pred_label = outs.max(dim=1)[1].item()
assert pred_label == true_label
verified=analyze(net, inputs, eps, true_label,verbose=VERBOSE)
if verified:
print('verified')
else:
print('not verified')
end = time.time()
if VERBOSE>0:
print("time passed: %f s" % (end - start))
#! TODO remove adverserial example search
Adv_exmp_found=runPGD(net, inputs, eps, true_label)
print("Adverserial example found : %s" %Adv_exmp_found)
return verified,start-end
y = y_batch[i]
xprime = pgd_untargeted(model, x, y, k, eps, eps_step, device,
clip_min, clip_max, **kwargs)
xprime_batch_list.append(xprime)
xprime_batch_tensor = torch.stack(xprime_batch_list)
assert x_batch.size() == xprime_batch_tensor.size()
return xprime_batch_tensor
if __name__ == "__main__":
k = 10000000
netname = 'fc1'
spec = "../test_cases/" + netname + "/img0_0.06000.txt"
if netname == 'fc1':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 10]).to(DEVICE)
elif netname == 'fc2':
net = FullyConnected(DEVICE, INPUT_SIZE, [50, 50, 10]).to(DEVICE)
elif netname == 'fc3':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
elif netname == 'fc4':
net = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 10]).to(DEVICE)
elif netname == 'fc5':
net = FullyConnected(DEVICE, INPUT_SIZE,
[400, 200, 100, 100, 10]).to(DEVICE)
elif netname == 'conv1':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1)], [100, 10],
10).to(DEVICE)
elif netname == 'conv2':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)],
def main():
net = "conv1"
spec = "../test_cases/conv1/img0_0.00200.txt"
net_name = net
with open(spec, 'r') as f:
lines = [line[:-1] for line in f.readlines()]
true_label = int(lines[0])
pixel_values = [float(line) for line in lines[1:]]
eps = float(spec[:-4].split('/')[-1].split('_')[-1])
if net == 'fc1':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 10]).to(DEVICE)
elif net == 'fc2':
net = FullyConnected(DEVICE, INPUT_SIZE, [50, 50, 10]).to(DEVICE)
elif net == 'fc3':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
elif net == 'fc4':
net = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 10]).to(DEVICE)
elif net == 'fc5':
net = FullyConnected(DEVICE, INPUT_SIZE,
[400, 200, 100, 100, 10]).to(DEVICE)
elif net == 'conv1':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1)], [100, 10],
10).to(DEVICE)
elif net == 'conv2':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)],
[100, 10], 10).to(DEVICE)
elif net == 'conv3':
net = Conv(DEVICE, INPUT_SIZE, [(32, 3, 1, 1), (32, 4, 2, 1),
(64, 4, 2, 1)], [150, 10],
10).to(DEVICE)
elif net == 'conv4':
net = Conv(DEVICE, INPUT_SIZE, [(32, 4, 2, 1), (64, 4, 2, 1)],
[100, 100, 10], 10).to(DEVICE)
elif net == 'conv5':
net = Conv(DEVICE, INPUT_SIZE, [(16, 3, 1, 1), (32, 4, 2, 1),
(64, 4, 2, 1)], [100, 100, 10],
10).to(DEVICE)
net.load_state_dict(
torch.load('../mnist_nets/%s.pt' % net_name,
map_location=torch.device(DEVICE)))
inputs = torch.FloatTensor(pixel_values).view(1, 1, INPUT_SIZE,
INPUT_SIZE).to(DEVICE)
outs = net(inputs)
pred_label = outs.max(dim=1)[1].item()
assert pred_label == true_label
if analyze(net, inputs, eps, true_label):
print('verified')
else:
print('not verified')
def main():
parser = argparse.ArgumentParser(
description='Neural network verification using DeepZ relaxation')
parser.add_argument(
'--net',
type=str,
choices=[
'fc1', 'fc2', 'fc3', 'fc4', 'fc5', 'fc6', 'fc7', 'conv1', 'conv2',
'conv3'
],
required=True,
help='Neural network architecture which is supposed to be verified.')
parser.add_argument('--spec',
type=str,
required=True,
help='Test case to verify.')
args = parser.parse_args()
with open(args.spec, 'r') as f:
lines = [line[:-1] for line in f.readlines()]
true_label = int(lines[0])
pixel_values = [float(line) for line in lines[1:]]
eps = float(args.spec[:-4].split('/')[-1].split('_')[-1])
if args.net == 'fc1':
net = FullyConnected(DEVICE, INPUT_SIZE, [50, 10]).to(DEVICE)
abstract_net = AbstractFullyConnected(DEVICE, INPUT_SIZE,
[50, 10]).to(DEVICE)
elif args.net == 'fc2':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 50, 10]).to(DEVICE)
abstract_net = AbstractFullyConnected(DEVICE, INPUT_SIZE,
[100, 50, 10]).to(DEVICE)
elif args.net == 'fc3':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 10]).to(DEVICE)
abstract_net = AbstractFullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 10]).to(DEVICE)
elif args.net == 'fc4':
net = FullyConnected(DEVICE, INPUT_SIZE, [100, 100, 50, 10]).to(DEVICE)
abstract_net = AbstractFullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 50, 10]).to(DEVICE)
elif args.net == 'fc5':
net = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 10]).to(DEVICE)
abstract_net = AbstractFullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 10]).to(DEVICE)
elif args.net == 'fc6':
net = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 100, 10]).to(DEVICE)
abstract_net = AbstractFullyConnected(
DEVICE, INPUT_SIZE, [100, 100, 100, 100, 10]).to(DEVICE)
elif args.net == 'fc7':
net = FullyConnected(DEVICE, INPUT_SIZE,
[100, 100, 100, 100, 100, 10]).to(DEVICE)
abstract_net = AbstractFullyConnected(
DEVICE, INPUT_SIZE, [100, 100, 100, 100, 100, 10]).to(DEVICE)
elif args.net == 'conv1':
net = Conv(DEVICE, INPUT_SIZE, [(16, 3, 2, 1)], [100, 10],
10).to(DEVICE)
abstract_net = AbstractConv(DEVICE, INPUT_SIZE, [(16, 3, 2, 1)],
[100, 10], 10).to(DEVICE)
elif args.net == 'conv2':
net = Conv(DEVICE, INPUT_SIZE, [(16, 4, 2, 1), (32, 4, 2, 1)],
[100, 10], 10).to(DEVICE)
abstract_net = AbstractConv(DEVICE, INPUT_SIZE, [(16, 4, 2, 1),
(32, 4, 2, 1)],
[100, 10], 10).to(DEVICE)
elif args.net == 'conv3':
net = Conv(DEVICE, INPUT_SIZE, [(16, 4, 2, 1), (64, 4, 2, 1)],
[100, 100, 10], 10).to(DEVICE)
abstract_net = AbstractConv(DEVICE, INPUT_SIZE, [(16, 4, 2, 1),
(64, 4, 2, 1)],
[100, 100, 10], 10).to(DEVICE)
else:
assert False
# here we are loading the pre-trained net weights
net.load_state_dict(
torch.load('../mnist_nets/%s.pt' % args.net,
map_location=torch.device(DEVICE)))
abstract_net.load_weights(net)
if type(abstract_net) == AbstractConv:
#cast it to fully connected
print("Converting the network to fully connected...")
abstract_net = abstract_net.make_fully_connected(DEVICE).to(DEVICE)
print("Conversion done")
inputs = torch.FloatTensor(pixel_values).view(1, 1, INPUT_SIZE,
INPUT_SIZE).to(DEVICE)
outs = net(inputs)
pred_label = outs.max(dim=1)[1].item()
assert pred_label == true_label
if analyze(abstract_net, inputs, eps, true_label):
print('verified')
else:
print('not verified')
print("-" * 20)
