def learn_diff_drive_robot(visualize=False):
all_faults = [
'left_faster', 'left_slower', 'left_stuck', 'right_faster',
'right_slower', 'right_stuck'
]
wheel_inputs = [(0, 0), (0, 2), (2, 0), (2, 2), (0, -2), (2, -2), (-2, 0),
(-2, 2), (-2, -2)]
alphabet = list(wheel_inputs)
alphabet.extend(all_faults)
sul = StrongFaultRobot(upper_speed_limit=10)
eq_oracle = RandomWMethodEqOracle(alphabet,
sul,
walks_per_state=20,
walk_len=15)
learned_model = run_Lstar(alphabet, sul, eq_oracle, automaton_type='mealy')
if visualize:
visualize_automaton(learned_model, display_same_state_trans=False)
return learned_model
def learn_vending_machine(visualize=False):
sul = VendingMachineSUL()
alphabet = sul.alphabet
eq_oracle = RandomWMethodEqOracle(alphabet,
sul,
walks_per_state=50,
walk_len=20)
learned_model = run_Lstar(alphabet, sul, eq_oracle, automaton_type='mealy')
# Example of a error
sul = MealySUL(learned_model)
print(
sul.query((
'add_coin_0.2',
'add_coin_0.5',
'add_coin_0.2',
'add_coin_0.2',
'add_coin_0.2',
'add_coin_0.2',
)))
if visualize:
visualize_automaton(learned_model, display_same_state_trans=False)
return learned_model
def test_all_configuration_combinations(self):
angluin_example = get_Angluin_dfa()
alphabet = angluin_example.get_input_alphabet()
automata_type = ['dfa', 'mealy', 'moore']
closing_strategies = ['shortest_first', 'longest_first', 'single']
cex_processing = [None, 'longest_prefix', 'rs']
suffix_closedness = [True, False]
caching = [True, False]
for automata in automata_type:
for closing in closing_strategies:
for cex in cex_processing:
for suffix in suffix_closedness:
for cache in caching:
sul = DfaSUL(angluin_example)
random_walk_eq_oracle = RandomWalkEqOracle(alphabet, sul, 5000, reset_after_cex=True)
state_origin_eq_oracle = StatePrefixEqOracle(alphabet, sul, walks_per_state=10, walk_len=50)
tran_cov_eq_oracle = TransitionFocusOracle(alphabet, sul, num_random_walks=200, walk_len=30,
same_state_prob=0.3)
w_method_eq_oracle = WMethodEqOracle(alphabet, sul,
max_number_of_states=len(angluin_example.states))
random_W_method_eq_oracle = RandomWMethodEqOracle(alphabet, sul,
walks_per_state=10, walk_len=50)
bf_exploration_eq_oracle = BreadthFirstExplorationEqOracle(alphabet, sul, 3)
random_word_eq_oracle = RandomWordEqOracle(alphabet, sul)
cache_based_eq_oracle = CacheBasedEqOracle(alphabet, sul)
kWayStateCoverageEqOracle = KWayStateCoverageEqOracle(alphabet, sul)
oracles = [random_walk_eq_oracle, random_word_eq_oracle, random_W_method_eq_oracle,
kWayStateCoverageEqOracle, cache_based_eq_oracle, bf_exploration_eq_oracle,
tran_cov_eq_oracle, w_method_eq_oracle, state_origin_eq_oracle]
if not cache:
oracles.remove(cache_based_eq_oracle)
for oracle in oracles:
sul = DfaSUL(angluin_example)
oracle.sul = sul
learned_model = run_Lstar(alphabet, sul, oracle, automaton_type=automata,
closing_strategy=closing, suffix_closedness=suffix,
cache_and_non_det_check=cache, cex_processing=cex,
print_level=0)
is_eq = self.prove_equivalence(learned_model)
if not is_eq:
print(oracle, automata)
assert False
assert True
def learn_light_switch(visualize=False):
alphabet = ['press', 'increase_delay', 'fix_delay'] # 'fix_delay'
sul = LightSwitchSUL()
eq_oracle = RandomWMethodEqOracle(alphabet,
sul,
walks_per_state=20,
walk_len=15)
learned_model = run_Lstar(alphabet, sul, eq_oracle, automaton_type='moore')
if visualize:
visualize_automaton(learned_model, display_same_state_trans=False)
return learned_model
def random_dfa_example(alphabet_size, number_of_states, num_accepting_states=1):
"""
Generate a random DFA machine and learn it.
:param alphabet_size: size of the input alphabet
:param number_of_states: number of states in the generated DFA
:param num_accepting_states: number of accepting states
:return: DFA
"""
import string
from aalpy.SULs import DfaSUL
from aalpy.learning_algs import run_Lstar
from aalpy.oracles import StatePrefixEqOracle, TransitionFocusOracle, WMethodEqOracle, \
RandomWalkEqOracle, RandomWMethodEqOracle, BreadthFirstExplorationEqOracle, RandomWordEqOracle, \
CacheBasedEqOracle, UserInputEqOracle, KWayStateCoverageEqOracle, KWayTransitionCoverageEqOracle, PacOracle
from aalpy.utils import generate_random_dfa
assert num_accepting_states <= number_of_states
alphabet = list(string.ascii_letters[:26])[:alphabet_size]
random_dfa = generate_random_dfa(number_of_states, alphabet, num_accepting_states)
alphabet = list(string.ascii_letters[:26])[:alphabet_size]
# visualize_automaton(random_dfa, path='correct')
sul_dfa = DfaSUL(random_dfa)
# examples of various equivalence oracles
random_walk_eq_oracle = RandomWalkEqOracle(alphabet, sul_dfa, 5000)
state_origin_eq_oracle = StatePrefixEqOracle(alphabet, sul_dfa, walks_per_state=10, walk_len=50)
tran_cov_eq_oracle = TransitionFocusOracle(alphabet, sul_dfa, num_random_walks=200, walk_len=30,
same_state_prob=0.3)
w_method_eq_oracle = WMethodEqOracle(alphabet, sul_dfa, max_number_of_states=number_of_states)
pac_oracle = PacOracle(alphabet, sul_dfa)
random_W_method_eq_oracle = RandomWMethodEqOracle(alphabet, sul_dfa, walks_per_state=10, walk_len=50)
bf_exploration_eq_oracle = BreadthFirstExplorationEqOracle(alphabet, sul_dfa, 5)
random_word_eq_oracle = RandomWordEqOracle(alphabet, sul_dfa)
cache_based_eq_oracle = CacheBasedEqOracle(alphabet, sul_dfa)
user_based_eq_oracle = UserInputEqOracle(alphabet, sul_dfa)
kWayStateCoverageEqOracle = KWayStateCoverageEqOracle(alphabet, sul_dfa)
kWayTransitionCoverageEqOracle = KWayTransitionCoverageEqOracle(alphabet, sul_dfa)
learned_dfa = run_Lstar(alphabet, sul_dfa, random_W_method_eq_oracle, automaton_type='dfa',
cache_and_non_det_check=True, cex_processing='rs')
# visualize_automaton(learned_dfa)
return learned_dfa
def learn_gearbox(visualize=False):
alphabet = [
'press_clutch', 'release_clutch', 'put_in_reverse', 'increase_gear',
'decrease_gear'
]
sul = GearBoxSUL()
eq_oracle = RandomWMethodEqOracle(alphabet,
sul,
walks_per_state=2000,
walk_len=15)
learned_model = run_Lstar(alphabet, sul, eq_oracle, automaton_type='mealy')
if visualize:
visualize_automaton(learned_model, display_same_state_trans=False)
return learned_model
def learn_wind_turbine(visualize=False):
alphabet = [
'increase_speed', 'stop_turbine', 'unexpected_speed_increase',
'unexpected_slow_down'
]
sul = TurbineSUL()
eq_oracle = RandomWMethodEqOracle(alphabet,
sul,
walks_per_state=20,
walk_len=15)
learned_model = run_Lstar(alphabet, sul, eq_oracle, automaton_type='mealy')
if visualize:
visualize_automaton(learned_model, display_same_state_trans=False)
return learned_model
def learn_coffee_machine_mbd(visualize=False):
sul = FaultInjectedCoffeeMachineSUL()
alphabet = ['coin', 'button', 'coin_double_value', 'button_no_effect']
eq_oracle = RandomWMethodEqOracle(alphabet,
sul,
walks_per_state=5000,
walk_len=20)
learned_model = run_Lstar(alphabet,
sul,
eq_oracle,
automaton_type='mealy',
cache_and_non_det_check=False)
if visualize:
visualize_automaton(learned_model, display_same_state_trans=True)
return learned_model
def learn_language_of_coffee_machine_error(visualize=False):
sul = FaultyCoffeeMachineSULDFA()
alphabet = ['coin', 'button']
eq_oracle = RandomWMethodEqOracle(alphabet,
sul,
walks_per_state=5000,
walk_len=20)
learned_model = run_Lstar(alphabet,
sul,
eq_oracle,
automaton_type='dfa',
cache_and_non_det_check=True)
if visualize:
visualize_automaton(learned_model, display_same_state_trans=True)
return learned_model
def big_input_alphabet_example(input_alphabet_size=1000, automaton_depth=4):
"""
Small example where input alphabet can be huge and outputs are just true and false (DFA).
Args:
input_alphabet_size: size of input alphabet
automaton_depth: depth of alternating True/False paths in final automaton
Returns:
learned model
"""
from aalpy.base import SUL
from aalpy.learning_algs import run_Lstar
from aalpy.oracles import RandomWMethodEqOracle
class alternatingSUL(SUL):
def __init__(self):
super().__init__()
self.counter = 0
def pre(self):
self.counter = 0
def post(self):
pass
def step(self, letter):
if letter is None:
return False
out = letter % 2
self.counter = min(self.counter + 1, automaton_depth)
if self.counter % 2 == 1:
return not out
return out
input_al = list(range(input_alphabet_size))
sul = alternatingSUL()
eq_oracle = RandomWMethodEqOracle(input_al, sul)
model = run_Lstar(input_al, sul, eq_oracle, 'dfa', cache_and_non_det_check=False)
return model
def learn_crossroad(visualize=False):
sul = CrossroadSUL()
alphabet = sul.full_alphabet
eq_oracle = RandomWMethodEqOracle(alphabet,
sul,
walks_per_state=10,
walk_len=30)
learned_model = run_Lstar(alphabet,
sul,
eq_oracle,
automaton_type='mealy',
cache_and_non_det_check=False,
max_learning_rounds=10)
if visualize:
visualize_automaton(learned_model,
display_same_state_trans=False,
file_type="dot")
return learned_model
def test_eq_oracles(self):
angluin_example = get_Angluin_dfa()
alphabet = angluin_example.get_input_alphabet()
automata_type = ['dfa', 'mealy', 'moore']
for automata in automata_type:
sul = DfaSUL(angluin_example)
random_walk_eq_oracle = RandomWalkEqOracle(alphabet, sul, 5000, reset_after_cex=True)
state_origin_eq_oracle = StatePrefixEqOracle(alphabet, sul, walks_per_state=10, walk_len=50)
tran_cov_eq_oracle = TransitionFocusOracle(alphabet, sul, num_random_walks=200, walk_len=30,
same_state_prob=0.3)
w_method_eq_oracle = WMethodEqOracle(alphabet, sul, max_number_of_states=len(angluin_example.states))
random_W_method_eq_oracle = RandomWMethodEqOracle(alphabet, sul, walks_per_state=10, walk_len=50)
bf_exploration_eq_oracle = BreadthFirstExplorationEqOracle(alphabet, sul, 3)
random_word_eq_oracle = RandomWordEqOracle(alphabet, sul)
cache_based_eq_oracle = CacheBasedEqOracle(alphabet, sul)
kWayStateCoverageEqOracle = KWayStateCoverageEqOracle(alphabet, sul)
oracles = [random_walk_eq_oracle, random_word_eq_oracle, random_W_method_eq_oracle, w_method_eq_oracle,
kWayStateCoverageEqOracle, cache_based_eq_oracle, bf_exploration_eq_oracle, tran_cov_eq_oracle,
state_origin_eq_oracle]
for oracle in oracles:
sul = DfaSUL(angluin_example)
oracle.sul = sul
learned_model = run_Lstar(alphabet, sul, oracle, automaton_type=automata,
cache_and_non_det_check=True, cex_processing=None, print_level=0)
is_eq = self.prove_equivalence(learned_model)
if not is_eq:
print(oracle, automata)
assert False
assert True
def to_mdp():
eq_oracle = RandomWMethodEqOracle(alphabet, model_sul)
learned_model = run_Lstar(alphabet, model_sul, eq_oracle, 'moore')
# CC2640R2-no-feature-req.dot
# {'mtu_req', 'pairing_req',} have 0.3 percent chance of looping to initial state
moore_mdp_state_map = dict()
initial_mdp_state = None
for state in learned_model.states:
mdp_state = MdpState(state.state_id, state.output)
moore_mdp_state_map[state.prefix] = mdp_state
if not state.prefix:
initial_mdp_state = mdp_state
# moore_mdp_state_map['sink'] = MdpState('sink', 'NO_RESPONSE')
assert initial_mdp_state
for state in learned_model.states:
mdp_state = moore_mdp_state_map[state.prefix]
state_num = int(mdp_state.state_id[1:])
for i in alphabet:
reached_moore = state.transitions[i].prefix
# if i in {'pairing_req', 'mtu_req'} and mdp_state.output != moore_mdp_state_map[reached_moore].output:
if state_num % 2 == 0 and mdp_state.output != moore_mdp_state_map[
reached_moore].output:
mdp_state.transitions[i].append((mdp_state, 0.2))
mdp_state.transitions[i].append(
(moore_mdp_state_map[reached_moore], 0.8))
else:
mdp_state.transitions[i].append(
(moore_mdp_state_map[reached_moore], 1.))
mdp = Mdp(initial_mdp_state, list(moore_mdp_state_map.values()))
return mdp
def compare_automata(aut_1: DeterministicAutomaton,
aut_2: DeterministicAutomaton,
num_cex=10):
"""
Finds cases of non-conformance between first and second automaton. This is done by performing RandomW equivalence
check. It is possible that number of found counterexamples is smaller than num_cex, as no counterexample will be a
suffix of a previously found counterexample.
Args:
aut_1: first automaton
aut_2: second automaton
num_cex: max. number of searches for counterexamples
Returns:
A list of input sequences that revel different behaviour on both automata. Counterexamples are sorted by length.
"""
#
from aalpy.oracles import RandomWMethodEqOracle
type_map = {MooreMachine: MooreSUL, Dfa: DfaSUL, MealyMachine: MealySUL}
assert set(aut_1.get_input_alphabet()) == set(aut_2.get_input_alphabet())
input_al = aut_1.get_input_alphabet()
# larger automaton is used as hypothesis, as then test-cases will contain prefixes leading to states
# not in smaller automaton
base_automaton, test_automaton = (
aut_1, aut_2) if aut_1.size < aut_2.size else (aut_2, aut_1)
base_sul = type_map[type(base_automaton)](base_automaton)
# compute prefixes for all states of the test automaton (needed for advanced eq. oracle)
for state in test_automaton.states:
if not state.prefix:
state.prefix = test_automaton.get_shortest_path(
test_automaton.initial_state, state)
# setup the eq oracle
eq_oracle = RandomWMethodEqOracle(input_al,
base_sul,
walks_per_state=min(
100,
len(input_al) * 10),
walk_len=10)
found_cex = []
# to avoid near "infinite" loops due to while loop and set requirement
# that is, if you can only find 1 cex and all other cexs are suffixes of that cex, first while condition will never
# be reached
failsafe_counter = 0
failsafe_stopping = num_cex * 100
while len(found_cex) < num_cex or failsafe_counter == failsafe_stopping:
cex = eq_oracle.find_cex(test_automaton)
# if no counterexample can be found terminate the loop
if cex is None:
break
if cex not in found_cex:
found_cex.append(cex)
failsafe_counter += 1
found_cex.sort(key=len)
return found_cex
def run_comparison(example, train=True, num_layers=2, hidden_dim=50, rnn_class=GRUNetwork,
insufficient_testing=False, verbose=False):
rnn, alphabet, train_set = train_or_load_rnn(example, num_layers=num_layers, hidden_dim=hidden_dim,
rnn_class=rnn_class, train=train)
# initial examples for Weiss et Al
all_words = sorted(list(train_set.keys()), key=lambda x: len(x))
pos = next((w for w in all_words if rnn.classify_word(w) is True), None)
neg = next((w for w in all_words if rnn.classify_word(w) is False), None)
starting_examples = [w for w in [pos, neg] if None is not w]
# Extract Automaton Using White-Box eq. query
rnn.renew()
if verbose:
print('---------------------------------WHITE BOX EXTRACTION--------------------------------------------------')
else:
blockPrint()
start_white_box = time.time()
dfa_weiss = extract(rnn, time_limit=500, initial_split_depth=10, starting_examples=starting_examples)
time_white_box = time.time() - start_white_box
# Make sure that internal states are back to initial
rnn.renew()
if verbose:
print('---------------------------------BLACK BOX EXTRACTION--------------------------------------------------')
sul = RNN_BinarySUL_for_Weiss_Framework(rnn)
alphabet = list(alphabet)
# define the equivalence oracle
if insufficient_testing:
eq_oracle = RandomWordEqOracle(alphabet, sul, num_walks=100, min_walk_len=3, max_walk_len=12)
else:
eq_oracle = RandomWMethodEqOracle(alphabet, sul, walks_per_state=1000, walk_len=25)
if 'tomita' not in example:
eq_oracle = TransitionFocusOracle(alphabet, sul, num_random_walks=1000, walk_len=20)
start_black_box = time.time()
aalpy_dfa = run_Lstar(alphabet=alphabet, sul=sul, eq_oracle=eq_oracle, automaton_type='dfa', max_learning_rounds=10,
print_level=2 , cache_and_non_det_check=False, cex_processing='rs')
time_black_box = time.time() - start_black_box
enablePrint()
if insufficient_testing:
if len(aalpy_dfa.states) == len(dfa_weiss.Q):
translated_weiss_2_aalpy = Weiss_to_AALpy_DFA_format(dfa_weiss)
sul = DfaSUL(translated_weiss_2_aalpy)
eq_oracle = RandomWMethodEqOracle(alphabet, sul, walks_per_state=1000, walk_len=10)
cex = eq_oracle.find_cex(aalpy_dfa)
if not cex:
print(
'-------------------------WHITE-Box vs. BLACK-BOX WITH INSUFFICIENT TESTING -------------------------')
print('White-box and Black-box technique extracted the same automaton.')
print(f'White-box time: {round(time_white_box, 2)} seconds.')
print(f'Black-box time: {round(time_black_box, 2)} seconds.')
else:
verify_cex(aalpy_dfa, translated_weiss_2_aalpy, rnn, [cex])
return
if len(aalpy_dfa.states) != len(dfa_weiss.Q):
print('---------------------------------WHITE vs. BLACK BOX EXTRACTION----------------------------------------')
nn_props = F'{"GRU" if rnn_class == GRUNetwork else "LSTM"}_layers_{num_layers}_dim_{hidden_dim}'
print(f'Example : {example}')
print(f'Configuration : {nn_props}')
print(f"Number of states\n "
f"White-box extraction: {len(dfa_weiss.Q)}\n "
f"Black-box extraction: {len(aalpy_dfa.states)}")
translated_weiss_2_aalpy = Weiss_to_AALpy_DFA_format(dfa_weiss)
sul = DfaSUL(translated_weiss_2_aalpy)
eq_oracle = RandomWMethodEqOracle(alphabet, sul, walks_per_state=10000, walk_len=20)
if 'tomita' not in example:
eq_oracle = TransitionFocusOracle(alphabet, sul)
cex_set = []
for _ in range(10):
cex = eq_oracle.find_cex(aalpy_dfa)
if cex and cex not in cex_set:
cex_set.append(cex)
cex_set.sort(key=len)
# verify that the counterexamples are not spurios and find out which model is correct one
real_cex = verify_cex(aalpy_dfa, translated_weiss_2_aalpy, rnn, cex_set)
if not real_cex:
print('Spurious CEX')
assert False
#print('Few Counterexamples')
#print(' ', cex_set[:3])
else:
print('Size of both models: ', len(aalpy_dfa.states))
def accuracy_test():
ground_truth_model = load_automaton_from_file(
'TrainingDataAndAutomata/bp_depth4.dot', automaton_type='dfa')
input_al = ground_truth_model.get_input_alphabet()
output_al = [1, 0]
train_seq, train_labels = generate_data_from_automaton(ground_truth_model,
input_al,
num_examples=10000,
lens=(1, 2, 3, 5, 8,
10, 12, 15,
20, 25, 30))
x_train, y_train, x_test, y_test = split_train_validation(train_seq,
train_labels,
0.8,
uniform=True)
# Train all neural networks with same parameters, this can be configured to train with different parameters
rnn = RNNClassifier(input_al,
output_dim=len(output_al),
num_layers=2,
hidden_dim=50,
x_train=x_train,
y_train=y_train,
x_test=x_test,
y_test=y_test,
batch_size=32,
nn_type='GRU')
rnn.train(epochs=150, stop_acc=1.0, stop_epochs=2, verbose=1)
sul = RnnBinarySUL(rnn)
gt_sul = DfaSUL(ground_truth_model)
random_walk_eq_oracle = RandomWalkEqOracle(input_al,
sul,
num_steps=10000,
reset_prob=0.05)
random_word_eq_oracle = RandomWordEqOracle(input_al,
sul,
min_walk_len=5,
max_walk_len=25,
num_walks=1000)
random_w_eq_oracle = RandomWMethodEqOracle(input_al,
sul,
walks_per_state=200,
walk_len=25)
learned_model = run_Lstar(input_al,
sul,
random_word_eq_oracle,
automaton_type='dfa',
max_learning_rounds=5)
from random import choice, randint
random_tc = []
coverage_guided_tc = []
num_tc = 1000
for _ in range(num_tc):
random_tc.append(
tuple(choice(input_al) for _ in range(randint(10, 25))))
prefix = choice(learned_model.states).prefix
middle = tuple(choice(input_al) for _ in range(20))
suffix = choice(learned_model.characterization_set)
coverage_guided_tc.append(prefix + middle + suffix)
num_adv_random = 0
for tc in random_tc:
correct = gt_sul.query(tc)
trained = sul.query(tc)
if correct != trained:
num_adv_random += 1
num_adv_guided = 0
for tc in coverage_guided_tc:
correct = gt_sul.query(tc)
trained = sul.query(tc)
if correct != trained:
num_adv_guided += 1
print(f'Random sampling: {round((num_adv_random/num_tc)*100,2)}')
print(f'Guided sampling: {round((num_adv_guided/num_tc)*100,2)}')