def test_a_gt_b_localrobustness_unsat(self):
os.environ["OUTPUT_LAYER"] = "-1"
os.environ["SHIFT"] = "np.asarray([[100,0]], dtype=np.float32)"
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx")
phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py")
result = self.verifier.verify(dnn, phi)
self.assertEqual(result, UNSAT)
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx")
phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py")
result = self.verifier.verify(dnn, phi)
self.assertEqual(result, UNSAT)
os.environ["SHIFT"] = "np.asarray([[0,100]], dtype=np.float32)"
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx")
phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py")
result = self.verifier.verify(dnn, phi)
self.assertEqual(result, UNSAT)
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx")
phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py")
result = self.verifier.verify(dnn, phi)
self.assertEqual(result, UNSAT)
def test_const_one_ge1_unsat(self):
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "const_one.onnx")
phi = properties.parse(property_artifact_dir / "output_ge1_0.py")
result = self.verifier.verify(dnn, phi)
self.assertEqual(result, UNSAT)
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "const_one.onnx")
phi = properties.parse(property_artifact_dir / "output_ge1_1.py")
result = self.verifier.verify(dnn, phi)
self.assertEqual(result, UNSAT)
def test_const_one_localrobustness(self):
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "const_one.onnx")
phi = properties.parse(
property_artifact_dir / "regression_localrobustness_0.py"
)
result = self.verifier.verify(dnn, phi)
self.assertEqual(result, UNSAT)
def test_sum_gt_one_localrobustness_shift_right_unsat(self):
os.environ["SHIFT"] = "100"
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "sum_gt_one.onnx")
phi = properties.parse(
property_artifact_dir / "regression_localrobustness_0.py"
)
result = self.verifier.verify(dnn, phi)
self.assertEqual(result, UNSAT)
def test_const_zero_ge1_sat(self):
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "const_zero.onnx")
phi = properties.parse(property_artifact_dir / "output_ge1_0.py")
result = self.verifier.verify(dnn, phi)
if self.is_complete:
self.assertEqual(result, SAT)
else:
self.assertIn(result, [UNKNOWN, SAT])
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "const_zero.onnx")
phi = properties.parse(property_artifact_dir / "output_ge1_1.py")
result = self.verifier.verify(dnn, phi)
if self.is_complete:
self.assertEqual(result, SAT)
else:
self.assertIn(result, [UNKNOWN, SAT])
def test_a_gt_b_localrobustness_sat(self):
os.environ["OUTPUT_LAYER"] = "-1"
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx")
phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py")
result = self.verifier.verify(dnn, phi)
if self.is_complete:
self.assertEqual(result, SAT)
else:
self.assertIn(result, [UNKNOWN, SAT])
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx")
phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py")
result = self.verifier.verify(dnn, phi)
if self.is_complete:
self.assertEqual(result, SAT)
else:
self.assertIn(result, [UNKNOWN, SAT])
def test_sum_gt_one_localrobustness_no_shift_sat(self):
os.environ["SHIFT"] = "0"
for i in range(RUNS_PER_PROP):
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "sum_gt_one.onnx")
phi = properties.parse(
property_artifact_dir / "regression_localrobustness_0.py"
)
result = self.verifier.verify(dnn, phi)
if self.is_complete:
self.assertEqual(result, SAT)
else:
self.assertIn(result, [UNKNOWN, SAT])
def test_mnist_relu_3_50(self):
verifiers = {
# "bab": bab, # too slow
"eran": eran,
"neurify": neurify,
# "planet": planet, # too slow
# "reluplex": reluplex, # too slow
}
for i in range(RUNS_PER_PROP):
os.environ["SEED"] = str(i)
results = []
for name, verifier in verifiers.items():
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "mnist_relu_3_50.onnx")
phi = properties.parse(property_artifact_dir /
"mnist_localrobustness_rand.py")
result = verifier.verify(dnn, phi)
self.check_results(result, results)
def test_random_residual_0(self):
os.environ["OUTPUT_LAYER"] = "-1"
verifiers = {"eran": eran, "planet": planet}
for epsilon in [0.01, 0.1, 0.5, 1.0]:
os.environ["EPSILON"] = str(epsilon)
for i in range(RUNS_PER_PROP):
os.environ["SEED"] = str(i)
results = []
for name, verifier in verifiers.items():
self.reset_property_context()
dnn = nn.parse(network_artifact_dir /
"random_residual_0.onnx")
phi = properties.parse(property_artifact_dir /
"localrobustness.py")
if name == "eran":
result = verifier.verify(dnn, phi, domain="refinezono")
else:
result = verifier.verify(dnn, phi)
self.check_results(result, results)
def test_random_fc_2(self):
os.environ["OUTPUT_LAYER"] = "-1"
verifiers = {
"bab": bab,
"eran": eran,
"neurify": neurify,
"planet": planet,
"reluplex": reluplex,
}
for epsilon in [0.01, 0.1, 0.5, 1.0]:
os.environ["EPSILON"] = str(epsilon)
for i in range(RUNS_PER_PROP):
os.environ["SEED"] = str(i)
results = []
for name, verifier in verifiers.items():
self.reset_property_context()
dnn = nn.parse(network_artifact_dir / "random_fc_2.onnx")
phi = properties.parse(property_artifact_dir /
"localrobustness.py")
result = verifier.verify(dnn, phi)
self.check_results(result, results)
