def test_mark_paths(self): at = AddTree() t = at.add_tree() t.split(t.root(), 0, 2) t.split(t.left(t.root()), 0, 1) t.split(t.right(t.root()), 0, 3) t.set_leaf_value(t.left(t.left(t.root())), 0.1) t.set_leaf_value(t.right(t.left(t.root())), 0.2) t.set_leaf_value(t.left(t.right(t.root())), 0.3) t.set_leaf_value(t.right(t.right(t.root())), 0.4) #print(at) dt = DomTree([(at, {0: RealDomain(0, 2)}), (at, {})]) l0 = dt.get_leaf(dt.tree().root()) v = Verifier(l0, Backend()) v.add_constraint(in_domain_constraint(v, l0.get_domains(0), instance=0)) v.add_constraint(v.xvar(0, instance=0) == v.xvar(0, instance=1)) self.assertFalse(l0.is_reachable(0, 0, 2)) self.assertTrue(l0.is_reachable(1, 0, 2)) v.instance(1).mark_unreachable_paths(0) v.add_constraint(v.xvar(0, instance=1) < 1.0) self.assertTrue(l0.is_reachable(0, 0, 4)) self.assertTrue(l0.is_reachable(1, 0, 4)) v.instance(0).mark_unreachable_paths(0, only_feat_id=999) v.instance(1).mark_unreachable_paths(0, only_feat_id=999) self.assertTrue(l0.is_reachable(0, 0, 4)) # no effect, wrong feat_id self.assertTrue(l0.is_reachable(1, 0, 4)) v.instance(0).mark_unreachable_paths(0, only_feat_id=0) v.instance(1).mark_unreachable_paths(0, only_feat_id=0) self.assertFalse(l0.is_reachable(0, 0, 4)) self.assertFalse(l0.is_reachable(1, 0, 4)) self.assertFalse(l0.is_reachable(1, 0, 2)) v.add_all_trees() #print(v._backend._solver) v.check() m = v.model() self.assertLess(m[0]["xs"][0], 1.0) self.assertGreaterEqual(m[0]["xs"][0], 0.0) self.assertEqual(m[0]["xs"][0], m[1]["xs"][0]) self.myAssertAlmostEqual(m[0]["ws"][0], 0.1)
def __call__(self, lk, check_paths): v = Verifier(lk, Backend()) v.add_constraint(v.fvar() > 5.0) v.add_constraint( z3.PbLe([(v.xvar(fid).get(), 1) for fid in v.instance(0).feat_ids()], 50)) return v
def __call__(self, lk, check_paths): v = Verifier(lk, Backend()) v.add_constraint(v.fvar(instance=0) < 0.0) v.add_constraint(v.xvar(0, instance=0) > 50) v.add_constraint(v.xvar(1, instance=0) < 50) # instances are exactly the same! for fid1, fid2 in zip( v.instance(0).feat_ids(), v.instance(1).feat_ids()): v.add_constraint( v.xvar(fid1, instance=0) == v.xvar(fid2, instance=1)) v.add_constraint( v.fvar(instance=1).get() - v.fvar(instance=0).get() < 9999) return v
def test_single_tree(self): at = AddTree() t = at.add_tree() t.split(t.root(), 0, 2) t.split(t.left(t.root()), 0, 1) t.split(t.right(t.root()), 0, 3) t.set_leaf_value(t.left(t.left(t.root())), 0.1) t.set_leaf_value(t.right(t.left(t.root())), 0.2) t.set_leaf_value(t.left(t.right(t.root())), 0.3) t.set_leaf_value(t.right(t.right(t.root())), 0.4) dt = DomTree(at, {}) l0 = dt.get_leaf(dt.tree().root()) v = Verifier(l0, Backend()) self.assertEqual(v._backend.check(v.fvar() < 0.0), Verifier.Result.SAT) v.add_tree(0) self.assertEqual(v._backend.check(v.fvar() < 0.0), Verifier.Result.UNSAT) self.assertEqual(v._backend.check(v.fvar() > 0.0), Verifier.Result.SAT) self.assertEqual(v._backend.check(v.fvar() < 0.41), Verifier.Result.SAT) self.assertEqual(v._backend.check(v.fvar() > 0.41), Verifier.Result.UNSAT) dt = DomTree(at, {0: RealDomain(1, 3)}) l0 = dt.get_leaf(dt.tree().root()) v = Verifier(l0, Backend()) v.add_constraint(v.xvar(0) < 2.0) v.add_tree(0) check = v.check( v.fvar() != t.get_leaf_value(t.right(t.left(t.root())))) self.assertEqual(check, Verifier.Result.UNSAT) dt = DomTree(at, {0: RealDomain(1, 3)}) l0 = dt.get_leaf(dt.tree().root()) v = Verifier(l0, Backend()) v.add_all_trees() self.assertEqual(v.check(v.fvar() < 0.0), Verifier.Result.UNSAT) self.assertEqual(v.check(v.fvar() > 0.0), Verifier.Result.SAT) self.assertEqual(v.check(v.fvar() < 0.41), Verifier.Result.SAT) self.assertEqual(v.check(v.fvar() > 0.41), Verifier.Result.UNSAT)
def __call__(self, lk, check_paths): v = Verifier(lk, Backend()) v.add_constraint(v.fvar() < 0.0) v.add_constraint(v.xvar(0) > 50) v.add_constraint(v.xvar(1) < 50) return v
def test_mark_paths2(self): at = AddTree.read("tests/models/xgb-img-easy.json") dt = DomTree(at, {}) l0 = dt.get_leaf(0) v = Verifier(l0, Backend()) v.add_constraint(v.fvar() < 0.0) v.add_constraint(v.xvar(0) > 50) v.add_constraint(v.xvar(0) <= 80) def test_reachable(m, l0): m(l0.is_reachable(0, 9, 109)) m(l0.is_reachable(0, 9, 117)) m(l0.is_reachable(0, 9, 95)) m(l0.is_reachable(0, 9, 104)) m(l0.is_reachable(0, 9, 83)) m(l0.is_reachable(0, 9, 70)) m(l0.is_reachable(0, 9, 52)) m(l0.is_reachable(0, 9, 37)) m(l0.is_reachable(0, 9, 5)) m(l0.is_reachable(0, 9, 29)) m(l0.is_reachable(0, 8, 96)) m(l0.is_reachable(0, 8, 3)) m(l0.is_reachable(0, 7, 23)) m(l0.is_reachable(0, 7, 15)) m(l0.is_reachable(0, 6, 118)) m(l0.is_reachable(0, 6, 98)) m(l0.is_reachable(0, 6, 37)) m(l0.is_reachable(0, 6, 5)) m(l0.is_reachable(0, 6, 29)) m(l0.is_reachable(0, 5, 112)) m(l0.is_reachable(0, 5, 90)) m(l0.is_reachable(0, 5, 66)) m(l0.is_reachable(0, 5, 3)) m(l0.is_reachable(0, 5, 37)) m(l0.is_reachable(0, 4, 84)) m(l0.is_reachable(0, 4, 70)) m(l0.is_reachable(0, 4, 53)) m(l0.is_reachable(0, 4, 37)) m(l0.is_reachable(0, 4, 5)) m(l0.is_reachable(0, 4, 29)) m(l0.is_reachable(0, 4, 23)) m(l0.is_reachable(0, 3, 108)) m(l0.is_reachable(0, 3, 92)) m(l0.is_reachable(0, 3, 84)) m(l0.is_reachable(0, 3, 78)) m(l0.is_reachable(0, 3, 3)) m(l0.is_reachable(0, 3, 49)) m(l0.is_reachable(0, 3, 43)) m(l0.is_reachable(0, 3, 37)) m(l0.is_reachable(0, 2, 96)) m(l0.is_reachable(0, 2, 80)) m(l0.is_reachable(0, 2, 66)) m(l0.is_reachable(0, 2, 3)) m(l0.is_reachable(0, 2, 49)) m(l0.is_reachable(0, 2, 37)) m(l0.is_reachable(0, 1, 94)) m(l0.is_reachable(0, 1, 82)) m(l0.is_reachable(0, 1, 66)) m(l0.is_reachable(0, 1, 3)) m(l0.is_reachable(0, 1, 35)) m(l0.is_reachable(0, 1, 23)) m(l0.is_reachable(0, 0, 98)) m(l0.is_reachable(0, 0, 86)) m(l0.is_reachable(0, 0, 29)) m(l0.is_reachable(0, 0, 5)) test_reachable(self.assertTrue, l0) for i in range(len(at)): v.instance(0).mark_unreachable_paths(i) test_reachable(self.assertFalse, l0)
def test_mnist_multi_instance(self): at = AddTree.read(f"tests/models/xgb-mnist-yis0-easy.json") dt = DomTree([(at, {}), (at, {})]) l0 = dt.get_leaf(dt.tree().root()) v = Verifier(l0, Backend()) v.add_all_trees(0) v.add_all_trees(1) v.add_constraint(v.fvar(0) > 5.0) # it is with high certainty X v.add_constraint(v.fvar(1) < -5.0) # it is with high certainty not X pbeq = [] for feat_id in AddTreeFeatureTypes(at).feat_ids(): bvar_name = f"b{feat_id}" v.add_bvar(bvar_name) bvar = v.bvar(bvar_name) xvar1 = v.xvar(feat_id, 0) xvar2 = v.xvar(feat_id, 1) v.add_constraint( z3.If(bvar.get(), xvar1.get() != xvar2.get(), xvar1.get() == xvar2.get())) pbeq.append((bvar.get(), 1)) N = 4 v.add_constraint(z3.PbLe(pbeq, N)) # at most N variables differ count = 0 uniques = set() img1_prev, img2_prev = None, None while count < 10: check = v.check() if not check.is_sat(): print("UNSAT") break model = v.model() img1 = np.zeros((28, 28)) img2 = np.zeros((28, 28)) hash1 = 127 hash2 = 91 for fid, x in model[0]["xs"].items(): p = np.unravel_index(fid, (28, 28)) if x is not None: img1[p[1], p[0]] = x hash1 = hash((hash1, fid, x)) for fid, x in model[1]["xs"].items(): p = np.unravel_index(fid, (28, 28)) if x is not None: img2[p[1], p[0]] = x hash2 = hash((hash2, fid, x)) uniques.add(hash((hash1, hash2))) if count > 0: diff1 = abs(img1 - img1_prev).sum() diff2 = abs(img2 - img2_prev).sum() print("mnist_multi_instance: norm difference:", diff1, diff2) self.assertGreater(diff1, 0.0) self.assertGreater(diff2, 0.0) img1_prev, img2_prev = img1, img2 # Ensure that the different pixels have different values in the next iteration # We do not care about the other pixels fam = v.model_family(model) fam_diff0 = {} fam_diff1 = {} for n, b in model["bs"].items(): if not b: continue i = int(n[1:]) p = np.unravel_index(i, (28, 28)) print("different pixel (bvar):", i, p, model[0]["xs"][i], model[1]["xs"][i]) if i in fam[0]: fam_diff0[i] = fam[0][i] if i in fam[1]: fam_diff1[i] = fam[1][i] v.add_constraint( not_in_domain_constraint(v, fam_diff0, 0) & not_in_domain_constraint(v, fam_diff1, 1)) count += 1 print("iteration", count, "uniques", len(uniques)) if count % 100 != 0: continue fig, (ax1, ax2) = plt.subplots(1, 2) ax1.imshow(img1, vmin=0, vmax=255) ax2.imshow(img2, vmin=0, vmax=255) ax1.set_title("instance 1: f={:.3f}".format(model[0]["f"])) ax2.set_title("instance 2: f={:.3f}".format(model[1]["f"])) for n, b in model["bs"].items(): if not b: continue i = int(n[1:]) p = np.unravel_index(i, (28, 28)) ax1.scatter([p[0]], [p[1]], marker=".", color="r") ax2.scatter([p[0]], [p[1]], marker=".", color="r") plt.show() self.assertEqual(count, 10) self.assertEqual(len(uniques), 10)
def test_two_trees(self): at = AddTree() t = at.add_tree() t.split(t.root(), 0, 2) t.split(t.left(t.root()), 0, 1) t.split(t.right(t.root()), 0, 3) t.set_leaf_value(t.left(t.left(t.root())), 0.1) t.set_leaf_value(t.right(t.left(t.root())), 0.2) t.set_leaf_value(t.left(t.right(t.root())), 0.3) t.set_leaf_value(t.right(t.right(t.root())), 0.4) t = at.add_tree() t.split(t.root(), 0, 2) t.split(t.left(t.root()), 1, 1) t.split(t.right(t.root()), 1, 3) t.set_leaf_value(t.left(t.left(t.root())), 0.1) t.set_leaf_value(t.right(t.left(t.root())), 0.2) t.set_leaf_value(t.left(t.right(t.root())), -0.3) t.set_leaf_value(t.right(t.right(t.root())), -0.4) dt = DomTree(at, {}) l0 = dt.get_leaf(dt.tree().root()) v = Verifier(l0, Backend()) v.add_all_trees() self.assertEqual(v.check(v.fvar() < -0.11), Verifier.Result.UNSAT) self.assertEqual(v.check(v.fvar() < -0.09), Verifier.Result.SAT) self.myAssertAlmostEqual(v.model()["ws"], [0.3, -0.4]) self.assertEqual(v.check(v.fvar() > -0.09), Verifier.Result.SAT) self.assertEqual(v.check(v.fvar() > 0.41), Verifier.Result.UNSAT) self.assertEqual(v.check(v.fvar() > 0.39), Verifier.Result.SAT) self.myAssertAlmostEqual(v.model()["ws"], [0.2, 0.2]) v = Verifier(l0, Backend()) v.add_all_trees() v.add_constraint(v.xvar(0) < 2.0) self.assertEqual(v.check(v.fvar() < -0.09), Verifier.Result.UNSAT) self.assertEqual(v.check(v.fvar() > 0.39), Verifier.Result.SAT) self.myAssertAlmostEqual(v.model()["ws"], [0.2, 0.2]) v = Verifier(l0, Backend()) v.add_all_trees() v.add_constraint(v.xvar(0) >= 2.0) self.assertEqual(v.check(v.fvar() < -0.09), Verifier.Result.SAT) self.myAssertAlmostEqual(v.model()["ws"], [0.3, -0.4]) self.assertEqual(v.check(v.fvar() > 0.39), Verifier.Result.UNSAT) dt = DomTree(at, {0: RealDomain(-math.inf, 2.0)}) l0 = dt.get_leaf(dt.tree().root()) v = Verifier(l0, Backend()) v.add_all_trees() self.assertEqual(v.check(v.fvar() < -0.09), Verifier.Result.UNSAT) self.assertEqual(v.check(v.fvar() > 0.39), Verifier.Result.SAT) self.myAssertAlmostEqual(v.model()["ws"], [0.2, 0.2]) dt = DomTree(at, {0: RealDomain(2.0, math.inf)}) l0 = dt.get_leaf(dt.tree().root()) v = Verifier(l0, Backend()) v.add_all_trees() self.assertEqual(v.check(v.fvar() < -0.09), Verifier.Result.SAT) self.myAssertAlmostEqual(v.model()["ws"], [0.3, -0.4]) self.assertEqual(v.check(v.fvar() > 0.39), Verifier.Result.UNSAT) v.add_constraint(v.xvar(1) < 2.0) self.assertEqual(v.check(v.fvar() < -0.09), Verifier.Result.UNSAT) self.assertEqual(v.check(v.fvar() < 0.01), Verifier.Result.SAT) model = v.model() #print(model) self.myAssertAlmostEqual(model["ws"], [0.3, -0.3]) #print(v.model_family(model)) v.add_constraint(not_in_domain_constraint(v, v.model_family(model), 0)) self.assertEqual(v.check(v.fvar() < 0.01), Verifier.Result.UNSAT)