def check_equivalence(prog_before, prog_after, allow_undef): # The equivalence check of the solver # For all input packets and possible table matches the programs should # be the same z3_prog_before, input_names_before, _ = prog_before z3_prog_after, input_names_after, _ = prog_after undef_violation = False try: z3_prog_before = z3.simplify(z3_prog_before) z3_prog_after = z3.simplify(z3_prog_after) # the equivalence equation log.debug("Simplifying equation...") tv_equiv = z3.simplify(z3_prog_before != z3_prog_after) except z3.Z3Exception as e: # Encountered an exception when trying to compare the formulas # There might be many reasons for this error_string = "Failed to compare Z3 formulas!\nReason: %s" % e error_string += "\nPROGRAM BEFORE\n" error_string += get_hdr_table(z3_prog_before, input_names_before) error_string += "\n\nPROGRAM AFTER\n" error_string += get_hdr_table(z3_prog_after, input_names_after) log.error(error_string) return util.EXIT_VIOLATION log.debug("Checking...") log.debug(z3.tactics()) t = z3.Then( z3.Tactic("simplify"), # z3.Tactic("distribute-forall"), # z3.Tactic("ackermannize_bv"), # z3.Tactic("bvarray2uf"), # z3.Tactic("card2bv"), # z3.Tactic("propagate-bv-bounds-new"), # z3.Tactic("reduce-bv-size"), # z3.Tactic("qe_rec"), z3.Tactic("smt"), ) solver = t.solver() log.debug(solver.sexpr()) ret = solver.check(tv_equiv) log.debug(tv_equiv) log.debug(ret) if allow_undef and ret == z3.sat: undef_violation = True # if we allow undefined changes we need to explicitly recheck log.info("Detected difference in undefined behavior. " "Rechecking while substituting undefined variables.") ret = undef_check(solver, z3_prog_before, z3_prog_after) if ret == z3.sat: print_validation_error(prog_before, prog_after, solver.model()) return util.EXIT_VIOLATION elif ret == z3.unknown: log.error("Solution unknown! There might be a problem...") return util.EXIT_VIOLATION else: if undef_violation: return util.EXIT_UNDEF return util.EXIT_SUCCESS
def build_test(config, main_formula, cond_tuple, pkt_range): permut_conds, avoid_conds, undefined_conds = cond_tuple # now we actually verify that we can find an input s = z3.Solver() # bind the output constant to the output of the main program output_const = z3.Const("output", main_formula.sort()) s.add(main_formula == output_const) undefined_matches = z3.And(*undefined_conds) s.add(undefined_matches) avoid_matches = z3.Not(z3.Or(*avoid_conds)) s.add(avoid_matches) # we need this tactic to find out which values will be undefined at the end # or which headers we expect to be invalid # the tactic effectively simplifies the formula to a single expression # under the constraints we have defined t = z3.Then(z3.Tactic("propagate-values"), z3.Tactic("ctx-solver-simplify"), z3.Tactic("elim-and")) # this is the test string we assemble stf_str = "" for permut in permut_conds: s.push() s.add(permut) log.info("Checking for solution...") ret = s.check() if ret == z3.sat: log.info("Found a solution!") # get the model m = s.model() # this does not work well yet... desperate hack # FIXME: Figure out a way to solve this, might not be solvable g = z3.Goal() g.add(main_formula == output_const, avoid_matches, undefined_matches, z3.And(*permut)) log.debug(z3.tactics()) log.info("Inferring simplified input and output") constrained_output = t.apply(g) log.info("Inferring dont-care map...") # FIXME: horrible output_var = constrained_output[0][0].children()[0] dont_care_map = get_dont_care_map(config, output_var, pkt_range) input_hdr = m[z3.Const(config["ingress_var"], output_const.sort())] output_hdr = m[output_const] log.debug("Output header: %s", output_hdr) log.debug("Input header: %s", input_hdr) flat_input = input_hdr.children()[pkt_range] flat_output = output_hdr.children()[pkt_range] stf_str += get_stf_str(flat_input, flat_output, dont_care_map) stf_str += "\n" else: # FIXME: This should be an error log.warning("No valid input could be found!") s.pop() # the final stf string lists all the interesting packets to test return stf_str
class Tactic: _mutate_probability = 0.02 _tactic_names = z3.tactics() _rand = random.Random() _rand.seed() def __init__(self, name): self.name = name self.params = {} pds = z3.Tactic(name).param_descrs() for i in range(pds.size()): pname = pds.get_name(i) self.params[pname] = Param(pname, None, pds.get_kind(pname)) def clone(self): res = Tactic(self.name) res.params = self.params.copy() return res def as_json(self): return json.dumps(self, cls=CustomJsonEncoder, indent=2) def mutate(self): for key, param in self.params.items(): if Tactic._rand.random() < Tactic._mutate_probability: self.params[key] = param._replace( value=random_param(param, Tactic._rand)) return self @staticmethod def random(): tactic_name = Tactic._rand.sample(Tactic._tactic_names, 1)[0] return Tactic(tactic_name) def __repr__(self): return "{0} -> {1}".format(self.name, self.params)
def check_equivalence(prog_before, prog_after, allow_undef): # The equivalence check of the solver # For all input packets and possible table matches the programs should # be the same try: # the equivalence equation log.debug("Simplifying equation...") tv_equiv = z3.simplify(prog_before != prog_after) except z3.Z3Exception as e: prog_before_simpl = z3.simplify(prog_before) prog_after_simpl = z3.simplify(prog_after) log.error("Failed to compare z3 formulas!\nReason: %s", e) log.error("PROGRAM BEFORE\n%s", prog_before_simpl) log.error("PROGRAM AFTER\n%s", prog_after_simpl) return util.EXIT_VIOLATION log.debug("Checking...") log.debug(z3.tactics()) t = z3.Then( z3.Tactic("simplify"), # z3.Tactic("distribute-forall"), # z3.Tactic("ackermannize_bv"), # z3.Tactic("bvarray2uf"), # z3.Tactic("card2bv"), # z3.Tactic("propagate-bv-bounds-new"), # z3.Tactic("reduce-bv-size"), # z3.Tactic("qe_rec"), z3.Tactic("smt"), ) s = t.solver() log.debug(s.sexpr()) ret = s.check(tv_equiv) log.debug(tv_equiv) log.debug(ret) if allow_undef and ret == z3.sat: prog_before = z3.simplify(prog_before) prog_after = z3.simplify(prog_after) log.info("Detected difference in undefined behavior. " "Rechecking with undefined variables ignored.") taints = set() log.info("Preprocessing...") prog_before, _ = substitute_taint(prog_before, taints) log.info("Checking...") tv_equiv = prog_before != prog_after if taints: tv_equiv = z3.ForAll(list(taints), tv_equiv) # check equivalence of the modified clause ret = s.check(tv_equiv) if ret == z3.sat: prog_before_simpl = z3.simplify(prog_before) prog_after_simpl = z3.simplify(prog_after) log.error("Detected an equivalence violation!") log.error("PROGRAM BEFORE\n%s", prog_before_simpl) log.error("PROGRAM AFTER\n%s", prog_after_simpl) log.error("Proposed solution:") log.error(s.model()) return util.EXIT_VIOLATION elif ret == z3.unknown: prog_before_simpl = z3.simplify(prog_before) prog_after_simpl = z3.simplify(prog_after) log.error("Solution unknown! There might be a problem...") return util.EXIT_VIOLATION else: return util.EXIT_SUCCESS
def perform_blackbox_test(config): out_dir = config["out_dir"] p4_input = config["p4_input"] if out_dir == OUT_DIR: out_dir = out_dir.joinpath(p4_input.stem) util.check_dir(out_dir) util.copy_file(p4_input, out_dir) config["out_dir"] = out_dir config["p4_input"] = p4_input # get the semantic representation of the original program z3_main_prog, result = get_semantics(config) if result != util.EXIT_SUCCESS: return result # now we actually verify that we can find an input s = z3.Solver() # we currently ignore all other pipelines and focus on the ingress pipeline main_formula = z3.simplify(z3_main_prog[config["pipe_name"]]) # this util might come in handy later. # z3.z3util.get_vars(main_formula) conditions = get_branch_conditions(main_formula) log.info(conditions) cond_tuple = dissect_conds(config, conditions) permut_conds, avoid_conds, undefined_conds = cond_tuple log.info("Computing permutations...") # FIXME: This does not scale well... # FIXME: This is a complete hack, use a more sophisticated method permuts = [[f(var) for var, f in zip(permut_conds, x)] for x in itertools.product([z3.Not, lambda x: x], repeat=len(permut_conds))] output_const = z3.Const("output", main_formula.sort()) # bind the output constant to the output of the main program s.add(main_formula == output_const) # all keys must be false for now # FIXME: Some of them should be usable log.info(15 * "#") log.info("Undefined conditions:") s.add(z3.And(*undefined_conds)) for cond in undefined_conds: log.info(cond) log.info("Conditions to avoid:") s.add(z3.Not(z3.Or(*avoid_conds))) for cond in avoid_conds: log.info(cond) log.info("Permissible permutations:") for cond in permuts: log.info(cond) log.info(15 * "#") for permut in permuts: s.push() s.add(permut) log.info("Checking for solution...") ret = s.check() if ret == z3.sat: log.info("Found a solution!") # get the model m = s.model() # this does not work well yet... desperate hack # FIXME: Figure out a way to solve this, might not be solvable avoid_matches = z3.Not(z3.Or(*avoid_conds)) undefined_matches = z3.And(*undefined_conds) permut_match = z3.And(*permut) g = z3.Goal() g.add(main_formula == output_const, avoid_matches, undefined_matches, permut_match) log.debug(z3.tactics()) t = z3.Then(z3.Tactic("propagate-values"), z3.Tactic("ctx-solver-simplify"), z3.Tactic("elim-and")) log.info("Inferring simplified input and output") constrained_output = t.apply(g) log.info("Inferring dont-care map...") output_var = constrained_output[0][0] dont_care_map = get_dont_care_map(config, output_var) result = check_with_stf(config, m, output_const, dont_care_map) if result != util.EXIT_SUCCESS: return result else: # FIXME: This should be an error log.warning("No valid input could be found!") s.pop() return result