def test_net_load():
""" Validate the loading of network such that it works on all 3k dataset samples. """
ds = c.CollisionData.load(device)
all_fpaths = list(Path(c.COLLISION_DIR).glob('*.rlv'))
print('Evaluating accuracies for each saved network (supposed to be the same):')
dom = DeeppolyDom() # which dom doesn't matter
for fpath in all_fpaths:
net = c.CollisionMPNet.load(fpath, dom, device)
outs = net(ds.inputs)
predicted = outs.argmax(dim=-1)
accuracy = len((predicted == ds.labels).nonzero(as_tuple=False)) / len(ds.labels)
print(f'-- Accuracy for {fpath.absolute().name}: {accuracy}')
assert accuracy >= 0.99
# inspect unsafe ones from last run, since all networks are the same, it doesn't matter
failed_bits = (predicted != ds.labels).nonzero(as_tuple=True)
failed_ins = ds.inputs[failed_bits]
failed_labels = ds.labels[failed_bits]
failed_outs = outs[failed_bits]
print(f'Failing on following inputs: {failed_ins}')
print(f'should be label: {failed_labels}')
print(f'with outpus: {failed_outs}')
return
def test_acas_input_optimizable():
dp = DeeppolyDom()
vi = IntervalDom()
_tc4(vi)
_tc4(dp)
return
def test_abstraction_soundness():
""" Validate that networks and inputs are abstracted correctly using implemented abstract domains.
However, it turns out that the usage of MaxPool1d makes it rather easy to trigger unsound violations.
see more details in tests/test_maxpool_soundness.py.
"""
all_fpaths = list(Path(c.COLLISION_DIR).glob('*.rlv'))
dom = DeeppolyDom()
net = c.CollisionMPNet.load(all_fpaths[0], dom,
device) # all nets are the same, just use one
unstable_cnts = 0
print('Evaluating abstraction correctness for the saved network:')
for fpath in all_fpaths:
prop = c.CollisionProp.load(fpath, dom)
lb, ub = prop.lbub(device)
pts = sample_points(lb, ub, 100)
out_conc = net(pts)
e = dom.Ele.by_intvl(lb, ub)
out_lb, out_ub = net(e).gamma()
threshold = 1e-5 # allow some numerical error
diff_lb = out_lb - out_conc
diff_ub = out_conc - out_ub
# print(diff_lb.max())
# print(diff_ub.max())
if diff_lb.max() >= threshold or diff_ub.max() >= threshold:
unstable_cnts += 1
print(
f'Network {fpath.absolute().name} found unsound cases (due to numerical instability because of MaxPool?)'
)
# print(diff_lb.max(), diff_ub.max())
print(f'-- Eventually, {unstable_cnts} networks seems to be unstable.')
return
def test_net_impl_1():
dp = DeeppolyDom()
vi = IntervalDom()
_tc1(vi)
_tc1(dp)
return
def test_load_and_save(ntimes: int = 10):
dom = DeeppolyDom()
prop = AcasProp.property2(dom)
all_fpaths = prop.applicable_net_paths()
for _ in range(ntimes):
fpath = random.choice(all_fpaths)
# print('Picked nnet file:', fpath)
net1, mins1, maxs1 = AcasNet.load_nnet(fpath, dom, device)
# print('Loaded:', net1)
with tempfile.TemporaryDirectory() as tmpdir:
tmp_path = Path(tmpdir, 'tmp.nnet')
net1.save_nnet(tmp_path, mins1, maxs1)
net2, mins2, maxs2 = AcasNet.load_nnet(tmp_path, dom, device)
assert mins1 == mins2
assert maxs1 == maxs2
assert len(net1.all_linears) == len(net2.all_linears)
for lin1, lin2 in zip(net1.all_linears, net2.all_linears):
assert torch.allclose(lin1.weight.data, lin2.weight.data)
assert torch.allclose(lin1.bias.data, lin2.bias.data)
return
def test_acas_net_optimizable():
dp = DeeppolyDom()
vi = IntervalDom()
_tc3(vi)
_tc3(dp)
return
def test_reluval_cex(nitems: int = 5):
""" Try to call ReluVal and collect its CEX. Validate that things are working. """
dom = DeeppolyDom()
reluval = ReluVal()
prop = AcasProp.property2(dom)
lb, ub = prop.lbub()
for npath in prop.applicable_net_paths()[:nitems]:
print('Using network from path', npath)
net, bound_mins, bound_maxs = AcasNet.load_nnet(npath, dom)
cexs = reluval.verify(lb, ub, net, task_name='2')
print(cexs)
# validation
for i in range(len(cexs)):
print('------ Validating cex', i)
cex = cexs[i:i + 1]
cex = net.normalize_inputs(cex, bound_mins, bound_maxs)
print('CEX:', cex)
with torch.no_grad():
out = net(cex)
print('Concrete Outs:', out)
assert out.argmax(dim=-1).item() == 0
absin = dom.Ele.by_intvl(cex, cex)
with torch.no_grad():
absout = net(absin)
print('Distance:', prop.safe_dist(absout))
print('------')
return
def setup_rest(self, args: argparse.Namespace):
""" Override this method to set up those not easily specified via command line arguments. """
# validation
assert not (args.no_pts and args.no_abs), 'training what?'
args.dom = {
'deeppoly': DeeppolyDom(),
'interval': IntervalDom()
}[args.dom]
if args.use_scheduler:
# having a scheduler does improve the accuracy quite a bit
args.scheduler_fn = lambda opti: ReduceLROnPlateau(
opti, factor=0.8, patience=10)
else:
args.scheduler_fn = lambda opti: None
return
def test_violation_example():
""" Demonstrate via a handcoded example. """
dom = DeeppolyDom()
err_eps = 1e-4
in_neurons = 1
fc1_neurons = 3
kernel_size, stride = 2, 1 # stride must be < kernel size, so as to allow overlapping
fc2_neurons = (fc1_neurons -
kernel_size) / stride + 1 # formula: (W - F + 2P) / S + 1
assert int(fc2_neurons) == fc2_neurons
fc2_neurons = int(fc2_neurons)
out_neurons = 1
lb = torch.tensor([[0.1]])
ub = torch.tensor([[0.12]])
pt = torch.tensor([[0.1010]])
fc1 = dom.Linear(in_neurons, fc1_neurons)
fc1.weight.data = torch.tensor([[0.9624], [-0.6785], [0.9087]])
fc1.bias.data = torch.tensor([0.3255, 0.7965, 0.6321])
relu = dom.ReLU()
mp = dom.MaxPool1d(kernel_size=kernel_size, stride=stride)
fc2 = dom.Linear(fc2_neurons, out_neurons, bias=False)
fc2.weight.data = torch.tensor([-0.6859, -0.4253])
def forward(x):
x = fc1(x)
x = relu(x)
x = x.unsqueeze(dim=1)
x = mp(x)
x = x.squeeze(dim=1)
x = fc2(x)
return x
print('in pt:', pt.squeeze().item())
print('in lb:', lb.squeeze().item())
print('in ub:', ub.squeeze().item())
e = dom.Ele.by_intvl(lb, ub)
out = forward(pt)
out_e = forward(e)
out_lb, out_ub = out_e.gamma()
assert not (out_lb <= out + err_eps).all()
print('out pt:', out.squeeze().item())
print('out lb:', out_lb.squeeze().item())
print('out ub:', out_ub.squeeze().item())
return
def test_andprop_conjoin():
""" Validate (manually..) that the AndProp is correct. """
dom = DeeppolyDom()
def _go(id):
props = id.applicable_props(dom)
ap = AndProp(props)
print('-- For network', id)
for p in props:
print('-- Has', p.name)
lb, ub = p.lbub()
print(' LB:', lb)
print(' UB:', ub)
lb, ub = ap.lbub()
print('-- All conjoined,', ap.name)
print(' LB:', lb)
print(' UB:', ub)
print(' Labels:', ap.labels)
print('Cnt:', len(lb))
for i in range(len(lb)):
print(' ', i, 'th piece, width:', ub[i] - lb[i],
f'area: {total_area(lb[[i]], ub[[i]]) :E}')
print()
return
''' <1, 1> is tricky, as it has many props;
<1, 9> is special, as it is different from many others;
Many others have prop1, prop2, prop3, prop4 would generate 3 pieces, in which prop1 and prop2 merged.
'''
# _go(AcasNetID(1, 1))
# _go(AcasNetID(1, 9))
# exit(0)
for id in AcasNetID.all_ids():
_go(id)
print('XL: Go manually check the outputs..')
return
safe_ratio = len(
safe_bits.nonzero(as_tuple=False)) / float(sample_size)
print('Iter', i, ': Safe ratio for point outputs:', safe_ratio)
if safe_ratio != 1.0:
spurious_bits = ~safe_bits
spurious_dist = safe_dist[spurious_bits]
print(
'\t',
f'spurious dists: [ {spurious_dist.min()} ~ {spurious_dist.max()} ]'
)
print('\t', 'spurious dists:', spurious_dist)
pass
return
for p in AcasProp.all_props(dom):
print(f'===== For {p.name} =====')
debug_unsafe_point(p)
print('\n')
return
if __name__ == '__main__':
from diffabs import DeeppolyDom
dom = DeeppolyDom()
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
inspect_net_props(dom)
# inspect_prop_pts(dom)
pass
def test_sample_violation():
""" It suffices to generate such violations by sampling.
Having both MaxPool1d and FC2 is necessary to reproduce the bug. FC1 must have bias to easily reproduce the
bug while FC2 may have no bias. Eps = 1e-4 is maximal magnitude to reproduce the bug because the weights and
input bounds are initialized small.
"""
dom = DeeppolyDom()
err_eps = 1e-4
in_neurons = 1
fc1_neurons = 3
kernel_size, stride = 2, 1
out_neurons = 1
lb = torch.tensor([[0.1]])
ub = torch.tensor([[0.12]])
# fixed
fc2_neurons = (fc1_neurons - kernel_size) / stride + 1
assert int(fc2_neurons) == fc2_neurons
fc2_neurons = int(fc2_neurons) # if using MaxPool1d
# fc2_neurons = fc1_neurons # if not using MaxPool1d
fc1 = dom.Linear(in_neurons, fc1_neurons, bias=True)
relu = dom.ReLU()
mp = dom.MaxPool1d(kernel_size=kernel_size, stride=stride)
fc2 = dom.Linear(fc2_neurons, out_neurons, bias=False)
def forward(x):
x = fc1(x)
x = relu(x)
x = x.unsqueeze(dim=1)
x = mp(x)
x = x.squeeze(dim=1)
x = fc2(x)
return x
def reset_params():
fc1.reset_parameters()
fc2.reset_parameters()
return
k = 0
while True:
k += 1
reset_params()
pts = sample_points(lb, ub, 10000)
e = dom.Ele.by_intvl(lb, ub)
out_conc = forward(pts)
out_lb, out_ub = forward(e).gamma()
if (out_lb <= out_conc + err_eps).all():
continue
print(f'After {k} resets')
print('LB <= conc?', (out_lb <= out_conc + err_eps).all())
print('LB <= conc? detail', out_lb <= out_conc + err_eps)
bits = out_conc + err_eps <= out_lb
bits = bits.any(dim=1) # any dimension violation is sufficient
idxs = bits.nonzero().squeeze(dim=1)
idx = idxs[0] # just pick the 1st one to debug
viol_in = pts[[idx]]
viol_out = out_conc[[idx]]
print('conc in:', viol_in.squeeze().item())
print('out lb:', out_lb.squeeze().item())
print('out ub:', out_ub.squeeze().item())
print('conc out:', viol_out.squeeze().item())
torch.save([fc1, fc2, viol_in], 'error_ctx.pt')
break
return
def test_reluplex(logs_dir: str = './reluplex_logs/'):
""" Use property 2 logs from Reluplex for thoroughly examination.
Need to run Reluplex's script 2 first and prepare the logs in proper location.
:param logs_dir: directory of logs
"""
if not Path(logs_dir).is_dir():
print(f'{logs_dir} is not valid path for all logs.')
return
dom = DeeppolyDom()
def validate_normalize(dnn: AcasNet, cex: _CEX, mins, maxs):
""" Validate that the normalize() function works the same as NNET's normalizeInput/Output(). """
ni = torch.tensor([cex.inputs])
ni = dnn.normalize_inputs(ni, mins, maxs)
ni = ni[0].detach().numpy()
target = cex.inputs_normed
err = _errors(ni, target)
print('My PyTorch normalizing:', ni)
print('NNET normalizing:', target)
print('Error:', err)
return err
def validate_dnn(dnn, cex):
""" Validate that the DNN outputs the same result as NNET does. """
oi = torch.tensor([cex.inputs])
oo = dnn(oi)
oo = oo[0].detach().numpy()
target = cex.nnet_outputs_normed
err = _errors(oo, target)
print('PyTorch :', oo)
print('NNET C++:', target)
print('Error:', err)
return err
def validate_cex(c, log_path):
id = log_path[-7:-4] # e.g. 2_1 for property2_stats_2_1.txt
net_path = './acas_nets/ACASXU_run2a_%s_batch_2000.nnet' % id
print(net_path)
dnn, mins, maxs = AcasNet.load_nnet(net_path, dom)
err1 = validate_normalize(dnn, c, mins, maxs)
print('---')
err2 = validate_dnn(dnn, c)
print()
return err1, err2
reluplex = Reluplex()
log_files = [
fn for fn in os.listdir(logs_dir) if not fn.endswith('_summary.txt')
]
all_cexs = []
err1s = []
err2s = []
for log_name in log_files:
with open(Path(logs_dir, log_name), 'r') as f:
log_data = f.read()
cexs = reluplex.extract(log_data)
all_cexs.extend(cexs)
for c in cexs:
err1, err2 = validate_cex(c, log_name)
err1s.append(err1)
err2s.append(err2)
pass
print('Errors for normalization:')
for err in err1s:
print(err)
print('Avg:', sum(err1s) / len(err1s))
print('Errors for forward propagation:')
for err in err2s:
print(err)
print('Avg:', sum(err2s) / len(err2s))
print('Example:')
print(all_cexs[0])
return