def test_a_gt_b_localrobustness_unsat(self): os.environ["OUTPUT_LAYER"] = "-1" os.environ["SHIFT"] = "np.asarray([[100,0]], dtype=np.float32)" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT) os.environ["SHIFT"] = "np.asarray([[0,100]], dtype=np.float32)" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT)
def test_is_linear(self): dave_op_graph = parse(artifact_dir / "dave.onnx") self.assertTrue(dave_op_graph.is_linear) dronet_op_graph = parse(artifact_dir / "dronet.onnx") # falsified immediately at output layer self.assertFalse(dronet_op_graph.is_linear) # drop output layer, falsified slightly later self.assertFalse(dronet_op_graph[:-1, 0].is_linear)
def test_const_one_ge1_unsat(self): for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_one.onnx") phi = properties.parse(property_artifact_dir / "output_ge1_0.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_one.onnx") phi = properties.parse(property_artifact_dir / "output_ge1_1.py") result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT)
def test_resnet34__relu__random_inputs(self): pytorch_model = torch.nn.Sequential(self.pytorch_model.conv1, self.pytorch_model.bn1, self.pytorch_model.relu) model = parse(artifact_dir / "resnet34.onnx")[:4] tf_model = model.as_tf() self.compare_model_output(pytorch_model, tf_model)
def test_a_gt_b(self): op_graph = parse(artifact_dir / "a_gt_b.onnx") tf_model = op_graph.as_tf() for i in range(20): x = np.random.normal(size=(1, 2)).astype(np.float32) y = tf_model(x) assert (np.argmax(x, axis=1) == np.argmax(y, axis=1)).all()
def test_const_zero(self): op_graph = parse(artifact_dir / "const_zero.onnx") tf_model = op_graph.as_tf() for i in range(20): x = np.random.normal(size=(1, 2)).astype(np.float32) y = tf_model(x) assert (y == 0).all()
def test_sum_gt_one(self): op_graph = parse(artifact_dir / "sum_gt_one.onnx") output_op = op_graph.output_operations[0] # gemm prefix = OperationGraph([output_op.a]) op_graph_suffix = OperationGraph(op_graph.walk(DropPrefix(prefix))) x = np.ones((1, 1)).astype(np.float32) y = op_graph_suffix(x) y_expected = np.ones((1, 1)) self.assertTrue((np.abs(y_expected - y) <= THRESHOLD).all()) x = np.random.normal(size=(10, 1)).astype(np.float32) y = op_graph_suffix(x) y_expected = x.copy() self.assertTrue((np.abs(y_expected - y) <= THRESHOLD).all()) prefix_output_op = prefix.output_operations[0] # relu prefix = OperationGraph([prefix_output_op.x]) op_graph_suffix = OperationGraph(op_graph.walk(DropPrefix(prefix))) x = np.ones((1, 1)).astype(np.float32) y = op_graph_suffix(x) y_expected = np.ones((1, 1)) self.assertTrue((np.abs(y_expected - y) <= THRESHOLD).all()) x = np.random.normal(size=(10, 1)).astype(np.float32) y = op_graph_suffix(x) y_expected = np.clip(x, 0.0, None) self.assertTrue((np.abs(y_expected - y) <= THRESHOLD).all())
def test_sum_gt_one(self): op_graph = parse(artifact_dir / "sum_gt_one.onnx") tf_model = op_graph.as_tf() for i in range(10): x = np.random.normal(loc=0.0, size=(1, 10)).astype(np.float32) y = tf_model(x) x_sum = x.sum(axis=1) - 1 y_expected = np.maximum(x_sum, np.zeros_like(x_sum)) assert (np.abs(y - y_expected) < THRESHOLD).all()
def test_const_one_localrobustness(self): for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_one.onnx") phi = properties.parse( property_artifact_dir / "regression_localrobustness_0.py" ) result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT)
def test_const_zero_ge1_sat(self): for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_zero.onnx") phi = properties.parse(property_artifact_dir / "output_ge1_0.py") result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT]) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "const_zero.onnx") phi = properties.parse(property_artifact_dir / "output_ge1_1.py") result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT])
def test_slice(self): dave_op_graph = parse(artifact_dir / "dave.onnx") with self.assertRaises(TypeError) as cm: dave_op_graph["last_layer"] self.assertEqual( cm.exception.args[0], "Unsupported type for indexing operation graph: 'str'" ) with self.assertRaises(ValueError) as cm: dave_op_graph[::2] self.assertEqual( cm.exception.args[0], "Slicing does not support non-unit steps." ) with self.assertRaises(TypeError) as cm: dave_op_graph["start_layer", "last_layer"] self.assertEqual( cm.exception.args[0], "Unsupported type for slicing indices: 'str'" ) with self.assertRaises(TypeError) as cm: dave_op_graph[0, 0, 0] self.assertEqual( cm.exception.args[0], "Unsupported indexing expression (0, 0, 0)" ) with self.assertRaises(TypeError) as cm: dave_op_graph[:, "last_layer"] self.assertEqual( cm.exception.args[0], "Unsupported type for selecting operations: 'str'" ) sliced_dave = dave_op_graph[0:None] for i in range(5): x = np.random.randn(1, 100, 100, 3).astype(np.float32) y1 = dave_op_graph(x).item() y2 = sliced_dave(x).item() self.assertAlmostEqual(y1, y2) dronet_op_graph = parse(artifact_dir / "dronet.onnx") sliced_dronet = dronet_op_graph[:-1][0] for i in range(5): x = np.random.randn(1, 200, 200, 1).astype(np.float32) y1 = dronet_op_graph(x)[:, 0].item() y2 = sliced_dronet(x).item() self.assertAlmostEqual(y1, y2)
def test_dave(self): op_graph = parse(artifact_dir / "dave.onnx") tf_model_1 = op_graph.as_tf() tf_model_2 = op_graph[2:].simplify().as_tf() for i in range(5): x1 = np.random.randn(1, 100, 100, 3).astype(np.float32) y1 = tf_model_1(x1).item() x2 = x1.transpose((0, 3, 1, 2)) y2 = tf_model_2(x2).item() self.assertAlmostEqual(y1, y2)
def test_sum_gt_one_localrobustness_shift_right_unsat(self): os.environ["SHIFT"] = "100" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "sum_gt_one.onnx") phi = properties.parse( property_artifact_dir / "regression_localrobustness_0.py" ) result = self.verifier.verify(dnn, phi) self.assertEqual(result, UNSAT)
def test_resnet34__layer1block1__random_inputs(self): pytorch_model = torch.nn.Sequential( self.pytorch_model.conv1, self.pytorch_model.bn1, self.pytorch_model.relu, self.pytorch_model.maxpool, self.pytorch_model.layer1._modules["0"], ) model = parse(artifact_dir / "resnet34.onnx")[:12] tf_model = model.as_tf() self.compare_model_output(pytorch_model, tf_model, THRESHOLD * 10)
def test_a_gt_b_localrobustness_sat(self): os.environ["OUTPUT_LAYER"] = "-1" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_0.py") result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT]) for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "a_gt_b.onnx") phi = properties.parse(property_artifact_dir / "class_localrobustness_1.py") result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT])
def test_sum_gt_one_localrobustness_no_shift_sat(self): os.environ["SHIFT"] = "0" for i in range(RUNS_PER_PROP): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "sum_gt_one.onnx") phi = properties.parse( property_artifact_dir / "regression_localrobustness_0.py" ) result = self.verifier.verify(dnn, phi) if self.is_complete: self.assertEqual(result, SAT) else: self.assertIn(result, [UNKNOWN, SAT])
def test_dronet(self): op_graph = parse(artifact_dir / "dronet.onnx") tf_model_1 = op_graph.as_tf() tf_model_2 = op_graph[2:].simplify().as_tf() for i in range(5): x1 = np.random.randn(1, 200, 200, 1).astype(np.float32) y1 = tf_model_1(x1) x2 = x1.transpose((0, 3, 1, 2)) y2 = tf_model_2(x2) self.assertEqual(y1.shape, y2.shape) self.assertEqual(y1.shape[0], 1) self.assertAlmostEqual(y1[0, 0], y2[0, 0], places=6) self.assertAlmostEqual(y1[0, 1], y2[0, 1], places=6)
def test_resnet34__flatten__random_inputs(self): pytorch_model = torch.nn.Sequential( self.pytorch_model.conv1, self.pytorch_model.bn1, self.pytorch_model.relu, self.pytorch_model.maxpool, self.pytorch_model.layer1, self.pytorch_model.layer2, self.pytorch_model.layer3, self.pytorch_model.layer4, self.pytorch_model.avgpool, ) model = parse(artifact_dir / "resnet34.onnx")[:119] tf_model = model.as_tf() self.compare_model_output(pytorch_model, tf_model, THRESHOLD * 15)
def test_mnist_relu_3_50(self): verifiers = { # "bab": bab, # too slow "eran": eran, "neurify": neurify, # "planet": planet, # too slow # "reluplex": reluplex, # too slow } for i in range(RUNS_PER_PROP): os.environ["SEED"] = str(i) results = [] for name, verifier in verifiers.items(): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "mnist_relu_3_50.onnx") phi = properties.parse(property_artifact_dir / "mnist_localrobustness_rand.py") result = verifier.verify(dnn, phi) self.check_results(result, results)
def test_random_residual_0(self): os.environ["OUTPUT_LAYER"] = "-1" verifiers = {"eran": eran, "planet": planet} for epsilon in [0.01, 0.1, 0.5, 1.0]: os.environ["EPSILON"] = str(epsilon) for i in range(RUNS_PER_PROP): os.environ["SEED"] = str(i) results = [] for name, verifier in verifiers.items(): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "random_residual_0.onnx") phi = properties.parse(property_artifact_dir / "localrobustness.py") if name == "eran": result = verifier.verify(dnn, phi, domain="refinezono") else: result = verifier.verify(dnn, phi) self.check_results(result, results)
def test_a_gt_b(self): op_graph = parse(artifact_dir / "a_gt_b.onnx") output_op = op_graph.output_operations[0] # softmax prefix = OperationGraph([output_op.x]) op_graph_suffix = OperationGraph(op_graph.walk(DropPrefix(prefix))) x = np.random.normal(size=(10, 2)).astype(np.float32) y = op_graph_suffix(x) a_gt_b = x[:, 0] > x[:, 1] y_0_gt_1 = y[:, 0] > y[:, 1] self.assertTrue(np.all(~(a_gt_b ^ y_0_gt_1))) prefix_output_op = prefix.output_operations[0] # gemm prefix = OperationGraph([prefix_output_op.a]) op_graph_suffix = OperationGraph(op_graph.walk(DropPrefix(prefix))) x = np.clip(np.random.normal(size=(10, 2)), 0.0, None).astype(np.float32) y = op_graph_suffix(x) a_gt_b = x[:, 0] > x[:, 1] y_0_gt_1 = y[:, 0] > y[:, 1] self.assertTrue(np.all(~(a_gt_b ^ y_0_gt_1)))
def test_resnet50(self): op_graph = parse(artifact_dir / "resnet50.onnx") first_half = op_graph[:84] self.assertEqual(len(first_half.output_operations), 1, "Only 1 output operation expected.") self.assertIsInstance( first_half.output_operations[0], operations.Add, "Expected first slice to end with Add.", ) second_half = op_graph[84:] self.assertEqual(len(second_half.output_operations), 1, "Only 1 output operation expected.") self.assertIsInstance( second_half.output_operations[0], operations.Gemm, "Expected first slice to end with Gemm.", ) self.assert_composition_equals(op_graph, (first_half, second_half))
def test_random_fc_2(self): os.environ["OUTPUT_LAYER"] = "-1" verifiers = { "bab": bab, "eran": eran, "neurify": neurify, "planet": planet, "reluplex": reluplex, } for epsilon in [0.01, 0.1, 0.5, 1.0]: os.environ["EPSILON"] = str(epsilon) for i in range(RUNS_PER_PROP): os.environ["SEED"] = str(i) results = [] for name, verifier in verifiers.items(): self.reset_property_context() dnn = nn.parse(network_artifact_dir / "random_fc_2.onnx") phi = properties.parse(property_artifact_dir / "localrobustness.py") result = verifier.verify(dnn, phi) self.check_results(result, results)
def test_vgg16(self): op_graph = parse(artifact_dir / "vgg16.onnx") first_half = op_graph[:19] self.assertEqual(len(first_half.output_operations), 1, "Only 1 output operation expected.") self.assertIsInstance( first_half.output_operations[0], operations.Conv, "Expected first slice to end with Add.", ) second_half = op_graph[19:] self.assertEqual(len(second_half.output_operations), 1, "Only 1 output operation expected.") self.assertIsInstance( second_half.output_operations[0], operations.Gemm, "Expected first slice to end with Gemm.", ) self.assert_composition_equals(op_graph, (first_half, second_half), threshold=1.5 * THRESHOLD)
def test_resnet50_multi_output(self): op_graph = parse(artifact_dir / "resnet50.onnx") first_half = op_graph[:83] self.assertEqual(len(first_half.output_operations), 2, "Expected 2 output operations.") self.assertIsInstance( first_half.output_operations[0], operations.BatchNormalization, "Expected first output operation to be BatchNormalization.", ) self.assertIsInstance( first_half.output_operations[1], operations.BatchNormalization, "Expected second output operation to be BatchNormalization.", ) second_half = op_graph[84:] self.assertEqual(len(second_half.output_operations), 1, "Only 1 output operation expected.") self.assertIsInstance( second_half.output_operations[0], operations.Gemm, "Expected first slice to end with Gemm.", ) self.assert_composition_equals( op_graph, ( first_half, OperationGraph([ operations.Add( operations.Input((-1, 1024, 14, 14), np.float32), operations.Input((-1, 1024, 14, 14), np.float32), ) ]), second_half, ), )
def test_resnet34(self): op_graph = parse(artifact_dir / "resnet34.onnx") op_graph.pprint() self.assertEqual(self.stdout.getvalue(), RESNET34_TEXT)
def test_const_zero(self): op_graph = parse(artifact_dir / "const_zero.onnx") simplified_op_graph = op_graph.simplify() self.assert_close(op_graph, simplified_op_graph, input_shape=(1, 2))
def test_sum_gt_one(self): op_graph = parse(artifact_dir / "sum_gt_one.onnx") simplified_op_graph = op_graph.simplify() self.assert_close(op_graph, simplified_op_graph, input_shape=(1, 10))
def test_resnet34(self): op_graph = parse(artifact_dir / "resnet34.onnx") simplified_op_graph = op_graph.simplify() self.assert_close(op_graph, simplified_op_graph, threshold=10 * THRESHOLD)
def test_vgg16_random_inputs(self): pytorch_model = self.pytorch_model model = parse(artifact_dir / "vgg16.onnx") tf_model = model.as_tf() self.compare_model_output(pytorch_model, tf_model, THRESHOLD * 10)