def test_equivalence(
self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]:
if self.fbmethod == Feedback.ACC_SEQ:
self._update_afl_queue(fsm)
elif self.fbmethod == Feedback.W_TRACES:
self._update_afl_queue_wtraces(fsm)
elif self.fbmethod != Feedback.NONE:
assert False, f'unknown feedback method {self.fbmethod}'
equivalent = True
counterexample = None
if self.eqchecktype == EQCheckType.ERRORS or self.eqchecktype == EQCheckType.BOTH:
equivalent, counterexample = self._test_equivalence_helper(
fsm, self.aflutils.get_crashset())
if not equivalent:
return equivalent, tuple(counterexample)
if self.eqchecktype == EQCheckType.QUEUE or self.eqchecktype == EQCheckType.BOTH:
if self.enable_dtraces:
testset = self.aflutils.get_testset()
dset = get_distinguishing_set(fsm)
concatted = [tuple(a) + b for a, b in product(testset, dset)]
equivalent, counterexample = self._test_equivalence_helper(
fsm, concatted)
else:
equivalent, counterexample = self._test_equivalence_helper(
fsm, self.aflutils.get_testset())
if not equivalent:
return equivalent, tuple(counterexample)
return equivalent, counterexample
def test_equivalence(
self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]:
print("[info] Starting equivalence test")
# Don't bother with the distinguishing set for now
W = get_distinguishing_set(fsm)
P = get_state_cover_set(fsm)
print("[info] Got state cover set")
A = sorted([(x, ) for x in fsm.get_alphabet()])
equivalent = True
counterexample = None
for access_sequence in sorted(P, key=len, reverse=self.longest_first):
#print("[info] Trying access sequence:", access_sequence)
to_visit = deque()
to_visit.extend(A)
while len(to_visit) > 0:
cur = to_visit.popleft()
#print(cur)
# Check cache if this is invalid input
if access_sequence + cur in self.sul.invalid_cache:
continue
# Check cache if this is a known error
prefix, value = self.sul.error_cache.shortest_prefix(" ".join(
[str(x) for x in access_sequence + cur]))
if prefix is not None:
# Do check it tho
equivalent, counterexample = self._are_equivalent(
fsm, access_sequence + cur)
if not equivalent:
return equivalent, counterexample
continue
# If the test is of sufficient length, execute it
#if len(cur) == self.m:
#print("[Testing]", access_sequence + cur)
for w in [tuple()] + list(W):
equivalent, counterexample = self._are_equivalent(
fsm, access_sequence + cur + w)
if not equivalent:
return equivalent, counterexample
# If not, keep building
#else:
if len(cur) < self.m:
for a in A:
if access_sequence + cur + a not in self.sul.invalid_cache:
to_visit.append(cur + a)
return equivalent, counterexample
def test_equivalence(self, test_sul: SUL) -> Tuple[bool, Iterable]:
P = get_state_cover_set(test_sul)
W = get_distinguishing_set(test_sul, check=False)
equivalent = True
counterexample = None
for acc_seq in P:
equivalent, counterexample = self.blockcopy(acc_seq, test_sul, W)
if not equivalent:
return equivalent, counterexample
return equivalent, counterexample
def test_equivalence(
self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]:
# Get the NuSMV results
nusmv_results = self.nusmv.run_ltl_check(fsm)
# Get the counterexamples for the failed constraints:
nusmv_counterexamples = [
y[2] for y in filter(lambda x: x[1] == 'false', nusmv_results)
]
# Get distinguishing set
dset = get_distinguishing_set(fsm)
for nusmv_counterexample in nusmv_counterexamples:
prefix, loops = parse_nusmv_counterexample(nusmv_counterexample)
# Figure out the unroll counts for the loops
n_repeats = product(range(self.n_unrolls), repeat=len(loops))
for repeats in n_repeats:
# build testcase
cur_testcase = prefix
for idx, repeat in enumerate(repeats):
cur_testcase += (loops[idx] * repeat)
if self.enable_dset:
for dtrace in dset:
cur_testcase_w_dtrace = cur_testcase + tuple(dtrace)
if len(cur_testcase_w_dtrace
) > 0 and cur_testcase_w_dtrace[0] != '':
equivalent, counterexample = self._are_equivalent(
fsm, cur_testcase_w_dtrace)
if not equivalent:
return equivalent, counterexample
else:
if len(cur_testcase) > 0 and cur_testcase[0] != '':
equivalent, counterexample = self._are_equivalent(
fsm, cur_testcase)
if not equivalent:
return equivalent, counterexample
return True, None
def test_equivalence(
self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]:
print("Starting EQ test")
n = len(fsm.get_states())
m = self.m
assert m >= n, "hypothesis has more states than w-method bound"
depth = m - n
print('Attempting to determine distinguishing set')
W = get_distinguishing_set(fsm)
if len(W) < 1:
W.add(tuple())
print('distinguishing:', W)
P = get_state_cover_set(fsm)
print('state cover:', P)
X = fsm.get_alphabet() #set([(x,) for x in fsm.get_alphabet()])
equivalent = True
counterexample = None
order = sorted(range(1, depth + 1), reverse=self.longest_first)
for i in order:
print(i, '/', depth)
for p in P:
for x in product(X, repeat=i):
for w in W:
test_sequence = p + x + w
print(test_sequence)
equivalent, counterexample = self._are_equivalent(
fsm, test_sequence)
print("Test sequence: ", test_sequence)
if not equivalent:
print("COUNTEREXAMPLE:", counterexample)
return equivalent, counterexample
return equivalent, None
def _update_afl_queue_wtraces(self, fsm: Union[DFA, MealyMachine]):
# Make sure we have a folder to put our stuff in
queuepath = self.afl_dir.joinpath(f'output/{self.name}/queue')
queuepath.mkdir(exist_ok=True, parents=True)
state_cover = get_state_cover_set(fsm)
dset = get_distinguishing_set(fsm)
for acc_seq in state_cover:
acc_seq = tuple(acc_seq)
for dtrace in dset:
dtrace = tuple(dtrace)
whole_trace = acc_seq + dtrace
if whole_trace not in self.feedback_seqs:
filename = f'id:{str(len(self.feedback_seqs)).rjust(6, "0")}'
with open(queuepath.joinpath(filename), 'wb') as file:
for a in whole_trace:
file.write(bytes([int(a)]))
self.feedback_seqs.add(whole_trace)
def minimize(mm: MealyMachine):
dset = get_distinguishing_set(mm)
dset_outputs = get_dset_outputs(mm, dset)
# Find non-unique states:
state_map = {}
for state, outputs in dset_outputs.items():
if outputs not in state_map:
state_map[outputs] = [state]
else:
state_map[outputs].append(state)
for outputs, states in state_map.items():
if len(states) > 1:
og_state = states[0]
rest_states = states[1:]
states = mm.get_states()
for state in states:
for action, (other_state, output) in state.edges.items():
if other_state in rest_states:
state.edges[action] = og_state, output
return mm
def test_equivalence(
self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]:
print("[info] Starting equivalence test")
if self.m is not None:
n = len(fsm.get_states())
m = self.m
assert m >= n, "hypothesis has more states than w-method bound"
depth = m - n
else:
depth = self.horizon
print("Depth:", depth)
print("[info] Calculating distinguishing set")
W = get_distinguishing_set(fsm, check=False)
P = get_state_cover_set(fsm)
print("[info] Got state cover set")
# Ensure all access sequences have a counter
for p in P:
if p not in self.acc_seq_ce_counter:
self.acc_seq_ce_counter[p] = 0
A = sorted([(x, ) for x in fsm.get_alphabet()])
equivalent = True
counterexample = None
for access_sequence in self.acc_seq_order(P):
print("[info] Trying access sequence:", access_sequence)
to_visit = deque()
to_visit.extend(A)
while len(to_visit) > 0:
cur = to_visit.popleft()
# Grow the testing tree where possible
self.sul.reset()
sul_output_pre = self.sul.process_input(access_sequence + cur)
if sul_output_pre in self.stop_on or any([
sul_output_pre.startswith(x)
for x in self.stop_on_startswith
]):
self.stopping_set.add(access_sequence + cur)
#continue
elif len(cur) <= depth:
for a in A:
if access_sequence + cur + a not in self.stopping_set\
and access_sequence + cur + a not in P:
to_visit.append(cur + a)
# Perform the standard W-method tests
for w in W:
equivalent, counterexample = self._are_equivalent(
fsm, access_sequence + cur + w)
if not equivalent:
# find longest access sequence which overlaps with the current query
longest_acc_seq = None
cur_query = access_sequence + cur + w
for acc_seq in P:
if cur_query[0:len(acc_seq)] == acc_seq:
if longest_acc_seq is None or len(
acc_seq) > len(longest_acc_seq):
longest_acc_seq = acc_seq
print("Counterexample:", counterexample)
print("Longest acc seq:", longest_acc_seq)
self.acc_seq_ce_counter[longest_acc_seq] += 1
return equivalent, counterexample
# Nothing found for this access sequence:
self.acc_seq_ce_counter[access_sequence] = min(
0, self.acc_seq_ce_counter[access_sequence])
self.acc_seq_ce_counter[access_sequence] -= 1
return equivalent, counterexample
def test_equivalence(
self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]:
print("[info] Starting equivalence test")
if self.m is not None:
n = len(fsm.get_states())
m = self.m
assert m >= n, "hypothesis has more states than w-method bound"
depth = m - n
else:
depth = self.horizon
print("Depth:", depth)
print("[info] Calculating distinguishing set")
W = get_distinguishing_set(fsm, check=False)
P = get_state_cover_set(fsm)
print("[info] Got state cover set")
A = sorted([(x, ) for x in fsm.get_alphabet()])
equivalent = True
counterexample = None
acc_seq_tasks = deque(
zip(self.acc_seq_order(P), [
deque([a for a in A if a not in self.stopping_set])
for x in range(len(P))
]))
while len(acc_seq_tasks) > 0:
access_sequence, to_visit = acc_seq_tasks.popleft()
# bprint("[info] Trying access sequence:", access_sequence)
assert len(to_visit) > 0
cur = to_visit.popleft()
# Test without distinguishing sequence, important for early stopping
equivalent, counterexample = self._are_equivalent(
fsm, access_sequence + cur)
if not equivalent:
return equivalent, counterexample
if access_sequence + cur not in self.stopping_set:
# Basically the usual W-method tests:
for w in W:
equivalent, counterexample = self._are_equivalent(
fsm, access_sequence + cur + w)
if not equivalent:
return equivalent, counterexample
# If not, keep building
if len(cur) <= depth:
for a in A:
if access_sequence + cur + a not in self.stopping_set:
to_visit.append(cur + a)
if len(to_visit) > 0:
acc_seq_tasks.append((access_sequence, to_visit))
#else:
#print(access_sequence)
return equivalent, counterexample
def test_equivalence(
self, fsm: Union[DFA, MealyMachine]) -> Tuple[bool, Iterable]:
print("[info] Starting equivalence test")
if self.m is not None:
n = len(fsm.get_states())
m = self.m
assert m >= n, "hypothesis has more states than w-method bound"
depth = m - n
else:
depth = self.horizon
print("Depth:", depth)
print("[info] Calculating distinguishing set")
W = get_distinguishing_set(fsm, check=False)
P = get_state_cover_set(fsm)
print("[info] Got state cover set")
# Ensure all access sequences have a counter
for p in P:
if p not in self.acc_seq_ce_counter:
self.acc_seq_ce_counter[p] = 0
A = sorted([(x, ) for x in fsm.get_alphabet()])
equivalent = True
counterexample = None
for access_sequence in self.acc_seq_order(P):
print("[info] Trying access sequence:", access_sequence)
to_visit = deque()
to_visit.extend(A)
while len(to_visit) > 0:
cur = to_visit.popleft()
# Basically the usual W-method tests:
for w in W:
equivalent, counterexample = self._are_equivalent(
fsm, access_sequence + cur + w)
if not equivalent:
self.acc_seq_ce_counter[access_sequence] += 1
return equivalent, counterexample
# Also test without distinguishing sequence, important for early stopping
equivalent, counterexample = self._are_equivalent(
fsm, access_sequence + cur)
if not equivalent:
self.acc_seq_ce_counter[access_sequence] += 1
return equivalent, counterexample
# Cut this branch short?
if access_sequence + cur in self.stopping_set:
continue
# If not, keep building
#else:
if len(cur) <= depth:
for a in A:
if access_sequence + cur + a not in self.stopping_set:
to_visit.append(cur + a)
# Nothing found for this access sequence:
self.acc_seq_ce_counter[access_sequence] = min(
0, self.acc_seq_ce_counter[access_sequence])
self.acc_seq_ce_counter[access_sequence] -= 1
return equivalent, counterexample
else:
state_map[outputs].append(state)
for outputs, states in state_map.items():
if len(states) > 1:
og_state = states[0]
rest_states = states[1:]
states = mm.get_states()
for state in states:
for action, (other_state, output) in state.edges.items():
if other_state in rest_states:
state.edges[action] = og_state, output
return mm
if __name__ == "__main__":
A = ['a', 'b', 'c']
O = ['1', '2', '3']
mm = MakeRandomMealyMachine(200, A, O)
mm.render_graph()
dset = get_distinguishing_set(mm)
check_distinguishing_set(mm, dset)
mm = minimize(mm)
dset = get_distinguishing_set(mm)
check_distinguishing_set(mm, dset)
mm.render_graph()