def test_learning_based_on_accuracy_based_stopping(self):
example = 'first_grid'
mdp = load_automaton_from_file(f'../DotModels/MDPs/{example}.dot', automaton_type='mdp')
min_rounds = 10
max_rounds = 500
from aalpy.automata import StochasticMealyMachine
from aalpy.utils import model_check_experiment, get_properties_file, \
get_correct_prop_values
from aalpy.automata.StochasticMealyMachine import smm_to_mdp_conversion
aalpy.paths.path_to_prism = "C:/Program Files/prism-4.6/bin/prism.bat"
aalpy.paths.path_to_properties = "../Benchmarking/prism_eval_props/"
stopping_based_on_prop = (get_properties_file(example), get_correct_prop_values(example), 0.02)
input_alphabet = mdp.get_input_alphabet()
automaton_type = ['mdp', 'smm']
similarity_strategy = ['classic', 'normal', 'chi2']
cex_processing = [None, 'longest_prefix']
samples_cex_strategy = [None, 'bfs', 'random:200:0.3']
for aut_type in automaton_type:
for strategy in similarity_strategy:
for cex in cex_processing:
for sample_cex in samples_cex_strategy:
sul = StochasticMealySUL(mdp) if aut_type == 'smm' else MdpSUL(mdp)
eq_oracle = UnseenOutputRandomWalkEqOracle(input_alphabet, sul=sul, num_steps=200,
reset_prob=0.25,
reset_after_cex=True)
learned_model = run_stochastic_Lstar(input_alphabet=input_alphabet, eq_oracle=eq_oracle,
sul=sul, n_c=20,
n_resample=1000, min_rounds=min_rounds,
max_rounds=max_rounds,
automaton_type=aut_type, strategy=strategy,
cex_processing=cex,
samples_cex_strategy=sample_cex, target_unambiguity=0.99,
property_based_stopping=stopping_based_on_prop,
print_level=0)
if isinstance(learned_model, StochasticMealyMachine):
mdp = smm_to_mdp_conversion(learned_model)
else:
mdp = learned_model
results, diff = model_check_experiment(get_properties_file(example),
get_correct_prop_values(example), mdp)
for d in diff.values():
if d > stopping_based_on_prop[2]:
assert False
assert True
def learn_stochastic_system_and_do_model_checking(example, automaton_type='smm', n_c=20, n_resample=1000, min_rounds=10,
max_rounds=500, strategy='normal', cex_processing='longest_prefix',
stopping_based_on_prop=None, samples_cex_strategy=None):
import aalpy.paths
from aalpy.automata import StochasticMealyMachine
from aalpy.utils import model_check_experiment, get_properties_file, get_correct_prop_values
from aalpy.automata.StochasticMealyMachine import smm_to_mdp_conversion
aalpy.paths.path_to_prism = "C:/Program Files/prism-4.6/bin/prism.bat"
aalpy.paths.path_to_properties = "Benchmarking/prism_eval_props/"
learned_model = benchmark_stochastic_example(example, automaton_type, n_c, n_resample, min_rounds, max_rounds,
strategy,
cex_processing, stopping_based_on_prop, samples_cex_strategy)
if isinstance(learned_model, StochasticMealyMachine):
mdp = smm_to_mdp_conversion(learned_model)
else:
mdp = learned_model
values, diff = model_check_experiment(get_properties_file(example), get_correct_prop_values(example), mdp)
print('Value for each property:', [round(d * 100, 2) for d in values.values()])
print('Error for each property:', [round(d * 100, 2) for d in diff.values()])
automaton_type='smm',
n_c=n_c,
n_resample=n_resample,
min_rounds=min_rounds,
strategy=strat,
max_rounds=max_rounds,
return_data=True,
samples_cex_strategy=cex_stat,
print_level=1,
cex_processing=cex_proc,
property_based_stopping=stopping_data)
smm_2_mdp = smm_to_mdp_conversion(learned_smm)
mdp_results, mdp_err = model_check_experiment(
get_properties_file(exp_name),
get_correct_prop_values(exp_name), learned_mdp)
smm_results, smm_err = model_check_experiment(
get_properties_file(exp_name),
get_correct_prop_values(exp_name), smm_2_mdp)
properties_string_header = ",".join(
[f'{key}_val,{key}_err' for key in mdp_results.keys()])
property_string_mdp = ",".join([
f'{str(mdp_results[p])},{str(mdp_err[p])}'
for p in mdp_results.keys()
])
property_string_smm = ",".join([
f'{str(smm_results[p])},{str(smm_err[p])}'
for p in smm_results.keys()