def test_a_gt_b_localrobustness_unsat(self): os.environ["OUTPUT_LAYER"] = "-1" os.environ["SHIFT"] = "np.asarray([[100,0]], dtype=np.float32)" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT) os.environ["SHIFT"] = "np.asarray([[0,100]], dtype=np.float32)" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT)
def test_const_one_ge1_unsat(self): for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_one.onnx") phi = properties.parse(property_artifact_dir / "output_ge1_0.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_one.onnx") phi = properties.parse(property_artifact_dir / "output_ge1_1.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT)
def test_const_one_localrobustness(self): for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_one.onnx") phi = properties.parse( property_artifact_dir / "regression_localrobustness_0.py" ) result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT)
def test_sum_gt_one_localrobustness_shift_right_unsat(self): os.environ["SHIFT"] = "100" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "sum_gt_one.onnx") phi = properties.parse( property_artifact_dir / "regression_localrobustness_0.py" ) result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT)
def test_const_zero_ge1_sat(self): for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_zero.onnx") phi = properties.parse(property_artifact_dir / "output_ge1_0.py") result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT]) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_zero.onnx") phi = properties.parse(property_artifact_dir / "output_ge1_1.py") result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT])
def test_a_gt_b_localrobustness_sat(self): os.environ["OUTPUT_LAYER"] = "-1" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py") result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT]) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py") result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT])
def test_sum_gt_one_localrobustness_no_shift_sat(self): os.environ["SHIFT"] = "0" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "sum_gt_one.onnx") phi = properties.parse( property_artifact_dir / "regression_localrobustness_0.py" ) result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT])
def test_mnist_relu_3_50(self): verifiers = { # "bab": bab, # too slow "eran": eran, "neurify": neurify, # "planet": planet, # too slow # "reluplex": reluplex, # too slow } for i in range(RUNS_PER_PROP): os.environ["SEED"] = str(i) results = [] for name, verifier in verifiers.items(): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "mnist_relu_3_50.onnx") phi = properties.parse(property_artifact_dir / "mnist_localrobustness_rand.py") result = verifier.verify(dnn, phi) self.check_results(result, results)
def test_random_residual_0(self): os.environ["OUTPUT_LAYER"] = "-1" verifiers = {"eran": eran, "planet": planet} for epsilon in [0.01, 0.1, 0.5, 1.0]: os.environ["EPSILON"] = str(epsilon) for i in range(RUNS_PER_PROP): os.environ["SEED"] = str(i) results = [] for name, verifier in verifiers.items(): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "random_residual_0.onnx") phi = properties.parse(property_artifact_dir / "localrobustness.py") if name == "eran": result = verifier.verify(dnn, phi, domain="refinezono") else: result = verifier.verify(dnn, phi) self.check_results(result, results)
def test_random_fc_2(self): os.environ["OUTPUT_LAYER"] = "-1" verifiers = { "bab": bab, "eran": eran, "neurify": neurify, "planet": planet, "reluplex": reluplex, } for epsilon in [0.01, 0.1, 0.5, 1.0]: os.environ["EPSILON"] = str(epsilon) for i in range(RUNS_PER_PROP): os.environ["SEED"] = str(i) results = [] for name, verifier in verifiers.items(): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "random_fc_2.onnx") phi = properties.parse(property_artifact_dir / "localrobustness.py") result = verifier.verify(dnn, phi) self.check_results(result, results)