def test_equivalence( self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]: if self.fbmethod == Feedback.ACC_SEQ: self._update_afl_queue(fsm) elif self.fbmethod == Feedback.W_TRACES: self._update_afl_queue_wtraces(fsm) elif self.fbmethod != Feedback.NONE: assert False, f'unknown feedback method {self.fbmethod}' equivalent = True counterexample = None if self.eqchecktype == EQCheckType.ERRORS or self.eqchecktype == EQCheckType.BOTH: equivalent, counterexample = self._test_equivalence_helper( fsm, self.aflutils.get_crashset()) if not equivalent: return equivalent, tuple(counterexample) if self.eqchecktype == EQCheckType.QUEUE or self.eqchecktype == EQCheckType.BOTH: if self.enable_dtraces: testset = self.aflutils.get_testset() dset = get_distinguishing_set(fsm) concatted = [tuple(a) + b for a, b in product(testset, dset)] equivalent, counterexample = self._test_equivalence_helper( fsm, concatted) else: equivalent, counterexample = self._test_equivalence_helper( fsm, self.aflutils.get_testset()) if not equivalent: return equivalent, tuple(counterexample) return equivalent, counterexample
def test_equivalence( self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]: print("[info] Starting equivalence test") # Don't bother with the distinguishing set for now W = get_distinguishing_set(fsm) P = get_state_cover_set(fsm) print("[info] Got state cover set") A = sorted([(x, ) for x in fsm.get_alphabet()]) equivalent = True counterexample = None for access_sequence in sorted(P, key=len, reverse=self.longest_first): #print("[info] Trying access sequence:", access_sequence) to_visit = deque() to_visit.extend(A) while len(to_visit) > 0: cur = to_visit.popleft() #print(cur) # Check cache if this is invalid input if access_sequence + cur in self.sul.invalid_cache: continue # Check cache if this is a known error prefix, value = self.sul.error_cache.shortest_prefix(" ".join( [str(x) for x in access_sequence + cur])) if prefix is not None: # Do check it tho equivalent, counterexample = self._are_equivalent( fsm, access_sequence + cur) if not equivalent: return equivalent, counterexample continue # If the test is of sufficient length, execute it #if len(cur) == self.m: #print("[Testing]", access_sequence + cur) for w in [tuple()] + list(W): equivalent, counterexample = self._are_equivalent( fsm, access_sequence + cur + w) if not equivalent: return equivalent, counterexample # If not, keep building #else: if len(cur) < self.m: for a in A: if access_sequence + cur + a not in self.sul.invalid_cache: to_visit.append(cur + a) return equivalent, counterexample
def test_equivalence(self, test_sul: SUL) -> Tuple[bool, Iterable]: P = get_state_cover_set(test_sul) W = get_distinguishing_set(test_sul, check=False) equivalent = True counterexample = None for acc_seq in P: equivalent, counterexample = self.blockcopy(acc_seq, test_sul, W) if not equivalent: return equivalent, counterexample return equivalent, counterexample
def test_equivalence( self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]: # Get the NuSMV results nusmv_results = self.nusmv.run_ltl_check(fsm) # Get the counterexamples for the failed constraints: nusmv_counterexamples = [ y[2] for y in filter(lambda x: x[1] == 'false', nusmv_results) ] # Get distinguishing set dset = get_distinguishing_set(fsm) for nusmv_counterexample in nusmv_counterexamples: prefix, loops = parse_nusmv_counterexample(nusmv_counterexample) # Figure out the unroll counts for the loops n_repeats = product(range(self.n_unrolls), repeat=len(loops)) for repeats in n_repeats: # build testcase cur_testcase = prefix for idx, repeat in enumerate(repeats): cur_testcase += (loops[idx] * repeat) if self.enable_dset: for dtrace in dset: cur_testcase_w_dtrace = cur_testcase + tuple(dtrace) if len(cur_testcase_w_dtrace ) > 0 and cur_testcase_w_dtrace[0] != '': equivalent, counterexample = self._are_equivalent( fsm, cur_testcase_w_dtrace) if not equivalent: return equivalent, counterexample else: if len(cur_testcase) > 0 and cur_testcase[0] != '': equivalent, counterexample = self._are_equivalent( fsm, cur_testcase) if not equivalent: return equivalent, counterexample return True, None
def test_equivalence( self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]: print("Starting EQ test") n = len(fsm.get_states()) m = self.m assert m >= n, "hypothesis has more states than w-method bound" depth = m - n print('Attempting to determine distinguishing set') W = get_distinguishing_set(fsm) if len(W) < 1: W.add(tuple()) print('distinguishing:', W) P = get_state_cover_set(fsm) print('state cover:', P) X = fsm.get_alphabet() #set([(x,) for x in fsm.get_alphabet()]) equivalent = True counterexample = None order = sorted(range(1, depth + 1), reverse=self.longest_first) for i in order: print(i, '/', depth) for p in P: for x in product(X, repeat=i): for w in W: test_sequence = p + x + w print(test_sequence) equivalent, counterexample = self._are_equivalent( fsm, test_sequence) print("Test sequence: ", test_sequence) if not equivalent: print("COUNTEREXAMPLE:", counterexample) return equivalent, counterexample return equivalent, None
def _update_afl_queue_wtraces(self, fsm: Union[DFA, MealyMachine]): # Make sure we have a folder to put our stuff in queuepath = self.afl_dir.joinpath(f'output/{self.name}/queue') queuepath.mkdir(exist_ok=True, parents=True) state_cover = get_state_cover_set(fsm) dset = get_distinguishing_set(fsm) for acc_seq in state_cover: acc_seq = tuple(acc_seq) for dtrace in dset: dtrace = tuple(dtrace) whole_trace = acc_seq + dtrace if whole_trace not in self.feedback_seqs: filename = f'id:{str(len(self.feedback_seqs)).rjust(6, "0")}' with open(queuepath.joinpath(filename), 'wb') as file: for a in whole_trace: file.write(bytes([int(a)])) self.feedback_seqs.add(whole_trace)
def minimize(mm: MealyMachine): dset = get_distinguishing_set(mm) dset_outputs = get_dset_outputs(mm, dset) # Find non-unique states: state_map = {} for state, outputs in dset_outputs.items(): if outputs not in state_map: state_map[outputs] = [state] else: state_map[outputs].append(state) for outputs, states in state_map.items(): if len(states) > 1: og_state = states[0] rest_states = states[1:] states = mm.get_states() for state in states: for action, (other_state, output) in state.edges.items(): if other_state in rest_states: state.edges[action] = og_state, output return mm
def test_equivalence( self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]: print("[info] Starting equivalence test") if self.m is not None: n = len(fsm.get_states()) m = self.m assert m >= n, "hypothesis has more states than w-method bound" depth = m - n else: depth = self.horizon print("Depth:", depth) print("[info] Calculating distinguishing set") W = get_distinguishing_set(fsm, check=False) P = get_state_cover_set(fsm) print("[info] Got state cover set") # Ensure all access sequences have a counter for p in P: if p not in self.acc_seq_ce_counter: self.acc_seq_ce_counter[p] = 0 A = sorted([(x, ) for x in fsm.get_alphabet()]) equivalent = True counterexample = None for access_sequence in self.acc_seq_order(P): print("[info] Trying access sequence:", access_sequence) to_visit = deque() to_visit.extend(A) while len(to_visit) > 0: cur = to_visit.popleft() # Grow the testing tree where possible self.sul.reset() sul_output_pre = self.sul.process_input(access_sequence + cur) if sul_output_pre in self.stop_on or any([ sul_output_pre.startswith(x) for x in self.stop_on_startswith ]): self.stopping_set.add(access_sequence + cur) #continue elif len(cur) <= depth: for a in A: if access_sequence + cur + a not in self.stopping_set\ and access_sequence + cur + a not in P: to_visit.append(cur + a) # Perform the standard W-method tests for w in W: equivalent, counterexample = self._are_equivalent( fsm, access_sequence + cur + w) if not equivalent: # find longest access sequence which overlaps with the current query longest_acc_seq = None cur_query = access_sequence + cur + w for acc_seq in P: if cur_query[0:len(acc_seq)] == acc_seq: if longest_acc_seq is None or len( acc_seq) > len(longest_acc_seq): longest_acc_seq = acc_seq print("Counterexample:", counterexample) print("Longest acc seq:", longest_acc_seq) self.acc_seq_ce_counter[longest_acc_seq] += 1 return equivalent, counterexample # Nothing found for this access sequence: self.acc_seq_ce_counter[access_sequence] = min( 0, self.acc_seq_ce_counter[access_sequence]) self.acc_seq_ce_counter[access_sequence] -= 1 return equivalent, counterexample
def test_equivalence( self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]: print("[info] Starting equivalence test") if self.m is not None: n = len(fsm.get_states()) m = self.m assert m >= n, "hypothesis has more states than w-method bound" depth = m - n else: depth = self.horizon print("Depth:", depth) print("[info] Calculating distinguishing set") W = get_distinguishing_set(fsm, check=False) P = get_state_cover_set(fsm) print("[info] Got state cover set") A = sorted([(x, ) for x in fsm.get_alphabet()]) equivalent = True counterexample = None acc_seq_tasks = deque( zip(self.acc_seq_order(P), [ deque([a for a in A if a not in self.stopping_set]) for x in range(len(P)) ])) while len(acc_seq_tasks) > 0: access_sequence, to_visit = acc_seq_tasks.popleft() # bprint("[info] Trying access sequence:", access_sequence) assert len(to_visit) > 0 cur = to_visit.popleft() # Test without distinguishing sequence, important for early stopping equivalent, counterexample = self._are_equivalent( fsm, access_sequence + cur) if not equivalent: return equivalent, counterexample if access_sequence + cur not in self.stopping_set: # Basically the usual W-method tests: for w in W: equivalent, counterexample = self._are_equivalent( fsm, access_sequence + cur + w) if not equivalent: return equivalent, counterexample # If not, keep building if len(cur) <= depth: for a in A: if access_sequence + cur + a not in self.stopping_set: to_visit.append(cur + a) if len(to_visit) > 0: acc_seq_tasks.append((access_sequence, to_visit)) #else: #print(access_sequence) return equivalent, counterexample
def test_equivalence( self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]: print("[info] Starting equivalence test") if self.m is not None: n = len(fsm.get_states()) m = self.m assert m >= n, "hypothesis has more states than w-method bound" depth = m - n else: depth = self.horizon print("Depth:", depth) print("[info] Calculating distinguishing set") W = get_distinguishing_set(fsm, check=False) P = get_state_cover_set(fsm) print("[info] Got state cover set") # Ensure all access sequences have a counter for p in P: if p not in self.acc_seq_ce_counter: self.acc_seq_ce_counter[p] = 0 A = sorted([(x, ) for x in fsm.get_alphabet()]) equivalent = True counterexample = None for access_sequence in self.acc_seq_order(P): print("[info] Trying access sequence:", access_sequence) to_visit = deque() to_visit.extend(A) while len(to_visit) > 0: cur = to_visit.popleft() # Basically the usual W-method tests: for w in W: equivalent, counterexample = self._are_equivalent( fsm, access_sequence + cur + w) if not equivalent: self.acc_seq_ce_counter[access_sequence] += 1 return equivalent, counterexample # Also test without distinguishing sequence, important for early stopping equivalent, counterexample = self._are_equivalent( fsm, access_sequence + cur) if not equivalent: self.acc_seq_ce_counter[access_sequence] += 1 return equivalent, counterexample # Cut this branch short? if access_sequence + cur in self.stopping_set: continue # If not, keep building #else: if len(cur) <= depth: for a in A: if access_sequence + cur + a not in self.stopping_set: to_visit.append(cur + a) # Nothing found for this access sequence: self.acc_seq_ce_counter[access_sequence] = min( 0, self.acc_seq_ce_counter[access_sequence]) self.acc_seq_ce_counter[access_sequence] -= 1 return equivalent, counterexample
else: state_map[outputs].append(state) for outputs, states in state_map.items(): if len(states) > 1: og_state = states[0] rest_states = states[1:] states = mm.get_states() for state in states: for action, (other_state, output) in state.edges.items(): if other_state in rest_states: state.edges[action] = og_state, output return mm if __name__ == "__main__": A = ['a', 'b', 'c'] O = ['1', '2', '3'] mm = MakeRandomMealyMachine(200, A, O) mm.render_graph() dset = get_distinguishing_set(mm) check_distinguishing_set(mm, dset) mm = minimize(mm) dset = get_distinguishing_set(mm) check_distinguishing_set(mm, dset) mm.render_graph()