def test_learning_based_on_accuracy_based_stopping(self):
example = 'first_grid'
mdp = load_automaton_from_file(f'../DotModels/MDPs/{example}.dot', automaton_type='mdp')
min_rounds = 10
max_rounds = 500
from aalpy.automata import StochasticMealyMachine
from aalpy.utils import model_check_experiment, get_properties_file, \
get_correct_prop_values
from aalpy.automata.StochasticMealyMachine import smm_to_mdp_conversion
aalpy.paths.path_to_prism = "C:/Program Files/prism-4.6/bin/prism.bat"
aalpy.paths.path_to_properties = "../Benchmarking/prism_eval_props/"
stopping_based_on_prop = (get_properties_file(example), get_correct_prop_values(example), 0.02)
input_alphabet = mdp.get_input_alphabet()
automaton_type = ['mdp', 'smm']
similarity_strategy = ['classic', 'normal', 'chi2']
cex_processing = [None, 'longest_prefix']
samples_cex_strategy = [None, 'bfs', 'random:200:0.3']
for aut_type in automaton_type:
for strategy in similarity_strategy:
for cex in cex_processing:
for sample_cex in samples_cex_strategy:
sul = StochasticMealySUL(mdp) if aut_type == 'smm' else MdpSUL(mdp)
eq_oracle = UnseenOutputRandomWalkEqOracle(input_alphabet, sul=sul, num_steps=200,
reset_prob=0.25,
reset_after_cex=True)
learned_model = run_stochastic_Lstar(input_alphabet=input_alphabet, eq_oracle=eq_oracle,
sul=sul, n_c=20,
n_resample=1000, min_rounds=min_rounds,
max_rounds=max_rounds,
automaton_type=aut_type, strategy=strategy,
cex_processing=cex,
samples_cex_strategy=sample_cex, target_unambiguity=0.99,
property_based_stopping=stopping_based_on_prop,
print_level=0)
if isinstance(learned_model, StochasticMealyMachine):
mdp = smm_to_mdp_conversion(learned_model)
else:
mdp = learned_model
results, diff = model_check_experiment(get_properties_file(example),
get_correct_prop_values(example), mdp)
for d in diff.values():
if d > stopping_based_on_prop[2]:
assert False
assert True
def learn_stochastic_system_and_do_model_checking(example, automaton_type='smm', n_c=20, n_resample=1000, min_rounds=10,
max_rounds=500, strategy='normal', cex_processing='longest_prefix',
stopping_based_on_prop=None, samples_cex_strategy=None):
import aalpy.paths
from aalpy.automata import StochasticMealyMachine
from aalpy.utils import model_check_experiment, get_properties_file, get_correct_prop_values
from aalpy.automata.StochasticMealyMachine import smm_to_mdp_conversion
aalpy.paths.path_to_prism = "C:/Program Files/prism-4.6/bin/prism.bat"
aalpy.paths.path_to_properties = "Benchmarking/prism_eval_props/"
learned_model = benchmark_stochastic_example(example, automaton_type, n_c, n_resample, min_rounds, max_rounds,
strategy,
cex_processing, stopping_based_on_prop, samples_cex_strategy)
if isinstance(learned_model, StochasticMealyMachine):
mdp = smm_to_mdp_conversion(learned_model)
else:
mdp = learned_model
values, diff = model_check_experiment(get_properties_file(example), get_correct_prop_values(example), mdp)
print('Value for each property:', [round(d * 100, 2) for d in values.values()])
print('Error for each property:', [round(d * 100, 2) for d in diff.values()])
n_c, n_resample = n_c, n_resample
else:
if exp_name == 'first_grid':
n_c, n_resample = 20, 1000
elif exp_name == 'second_grid':
n_c, n_resample = 20, 2000
elif exp_name == 'shared_coin':
n_c, n_resample = 25, 2500
elif exp_name == 'slot_machine':
n_c, n_resample = 30, 5000
elif exp_name == 'mqtt':
n_c, n_resample = 20, 1000
elif exp_name == 'tcp':
n_c, n_resample = 20, 1000
stopping_data = (get_properties_file(exp_name),
get_correct_prop_values(exp_name), 0.02)
original_mdp = load_automaton_from_file(
path_to_dir + file, automaton_type='mdp')
input_alphabet = original_mdp.get_input_alphabet()
mdp_sul = MdpSUL(original_mdp)
eq_oracle = UnseenOutputRandomWordEqOracle(
input_alphabet,
mdp_sul,
num_walks=150,
min_walk_len=5,
max_walk_len=15,
reset_after_cex=True)
input_alphabet,
mdp_sul,
eq_oracle,
automaton_type='smm',
n_c=n_c,
n_resample=n_resample,
min_rounds=min_rounds,
strategy=strategy,
max_rounds=max_rounds,
return_data=True,
samples_cex_strategy="bfs")
smm_2_mdp = smm_to_mdp_conversion(learned_smm)
mdp_results, mdp_err = model_check_experiment(
get_properties_file(exp_name), get_correct_prop_values(exp_name),
learned_mdp)
smm_results, smm_err = model_check_experiment(
get_properties_file(exp_name), get_correct_prop_values(exp_name),
smm_2_mdp)
properties_string_header = ",".join(
[f'{key}_val,{key}_err' for key in mdp_results.keys()])
property_string_mdp = ",".join([
f'{str(mdp_results[p])},{str(mdp_err[p])}'
for p in mdp_results.keys()
])
property_string_smm = ",".join([
f'{str(smm_results[p])},{str(smm_err[p])}'
for p in smm_results.keys()
mdp_sul,
eq_oracle,
automaton_type='smm',
min_rounds=min_rounds,
strategy=strat,
max_rounds=max_rounds,
return_data=True,
samples_cex_strategy=cex_stat,
print_level=1,
cex_processing=cex_proc,
target_unambiguity=0.99)
smm_2_mdp = smm_to_mdp_conversion(learned_smm)
mdp_results, mdp_err = model_check_experiment(
get_properties_file(exp_name),
get_correct_prop_values(exp_name), learned_mdp)
smm_results, smm_err = model_check_experiment(
get_properties_file(exp_name),
get_correct_prop_values(exp_name), smm_2_mdp)
properties_string_header = ",".join(
[f'{key}_val,{key}_err' for key in mdp_results.keys()])
property_string_mdp = ",".join([
f'{str(mdp_results[p])},{str(mdp_err[p])}'
for p in mdp_results.keys()
])
property_string_smm = ",".join([
f'{str(smm_results[p])},{str(smm_err[p])}'
for p in smm_results.keys()
i = random.choice(input_alphabet)
o = mdp_sul.step(i)
sample.append((i, o))
data.append(sample)
mdp_sul.post()
learned_mdp = run_Alergia(data, automaton_type='mdp')
for s in data:
s.pop(0)
learned_smm = run_Alergia(data, automaton_type='smm')
smm_2_mdp = smm_to_mdp_conversion(learned_smm)
mdp_results, mdp_err = model_check_experiment(get_properties_file(exp_name),
get_correct_prop_values(exp_name), learned_mdp)
smm_results, smm_err = model_check_experiment(get_properties_file(exp_name),
get_correct_prop_values(exp_name), smm_2_mdp)
print(learned_mdp.size, learned_smm.size, smm_2_mdp.size)
print(f'-------{exp_name}---------')
print(f'MDP Error: {mdp_err}')
print(f'SMM Error: {smm_err}')
smm_diff = {}
for key, val in mdp_err.items():
if key not in smm_err.keys() or smm_err[key] == 0:
continue
smm_diff[key] = round(smm_err[key] - val, 2)
print(f'SMM differance: {smm_diff}')
