def test_learning_based_on_accuracy_based_stopping(self):
example = 'first_grid'
mdp = load_automaton_from_file(f'../DotModels/MDPs/{example}.dot', automaton_type='mdp')
min_rounds = 10
max_rounds = 500
from aalpy.automata import StochasticMealyMachine
from aalpy.utils import model_check_experiment, get_properties_file, \
get_correct_prop_values
from aalpy.automata.StochasticMealyMachine import smm_to_mdp_conversion
aalpy.paths.path_to_prism = "C:/Program Files/prism-4.6/bin/prism.bat"
aalpy.paths.path_to_properties = "../Benchmarking/prism_eval_props/"
stopping_based_on_prop = (get_properties_file(example), get_correct_prop_values(example), 0.02)
input_alphabet = mdp.get_input_alphabet()
automaton_type = ['mdp', 'smm']
similarity_strategy = ['classic', 'normal', 'chi2']
cex_processing = [None, 'longest_prefix']
samples_cex_strategy = [None, 'bfs', 'random:200:0.3']
for aut_type in automaton_type:
for strategy in similarity_strategy:
for cex in cex_processing:
for sample_cex in samples_cex_strategy:
sul = StochasticMealySUL(mdp) if aut_type == 'smm' else MdpSUL(mdp)
eq_oracle = UnseenOutputRandomWalkEqOracle(input_alphabet, sul=sul, num_steps=200,
reset_prob=0.25,
reset_after_cex=True)
learned_model = run_stochastic_Lstar(input_alphabet=input_alphabet, eq_oracle=eq_oracle,
sul=sul, n_c=20,
n_resample=1000, min_rounds=min_rounds,
max_rounds=max_rounds,
automaton_type=aut_type, strategy=strategy,
cex_processing=cex,
samples_cex_strategy=sample_cex, target_unambiguity=0.99,
property_based_stopping=stopping_based_on_prop,
print_level=0)
if isinstance(learned_model, StochasticMealyMachine):
mdp = smm_to_mdp_conversion(learned_model)
else:
mdp = learned_model
results, diff = model_check_experiment(get_properties_file(example),
get_correct_prop_values(example), mdp)
for d in diff.values():
if d > stopping_based_on_prop[2]:
assert False
assert True
def stop_based_on_confidence(hypothesis,
property_based_stopping,
print_level=2):
"""
Args:
hypothesis: Markov decision process
property_based_stopping: a tuple (path to properties file, list of correct property values, max allowed error)
print_level: 2 or 3 if output of model checking is to be printed during learning
Returns:
True if absolute error for all properties is smaller then property_based_stopping[2]
"""
from aalpy.automata.StochasticMealyMachine import smm_to_mdp_conversion
path_2_prop = property_based_stopping[0]
correct_values = property_based_stopping[1]
error_bound = property_based_stopping[2]
model = hypothesis
if isinstance(hypothesis, StochasticMealyMachine):
model = smm_to_mdp_conversion(hypothesis)
res, diff = model_check_experiment(path_2_prop, correct_values, model)
if print_level >= 2:
print('Error for each property:',
[round(d * 100, 2) for d in diff.values()])
if not diff:
return False
for d in diff.values():
if d > error_bound:
return False
return True
def learn_stochastic_system_and_do_model_checking(example, automaton_type='smm', n_c=20, n_resample=1000, min_rounds=10,
max_rounds=500, strategy='normal', cex_processing='longest_prefix',
stopping_based_on_prop=None, samples_cex_strategy=None):
import aalpy.paths
from aalpy.automata import StochasticMealyMachine
from aalpy.utils import model_check_experiment, get_properties_file, get_correct_prop_values
from aalpy.automata.StochasticMealyMachine import smm_to_mdp_conversion
aalpy.paths.path_to_prism = "C:/Program Files/prism-4.6/bin/prism.bat"
aalpy.paths.path_to_properties = "Benchmarking/prism_eval_props/"
learned_model = benchmark_stochastic_example(example, automaton_type, n_c, n_resample, min_rounds, max_rounds,
strategy,
cex_processing, stopping_based_on_prop, samples_cex_strategy)
if isinstance(learned_model, StochasticMealyMachine):
mdp = smm_to_mdp_conversion(learned_model)
else:
mdp = learned_model
values, diff = model_check_experiment(get_properties_file(example), get_correct_prop_values(example), mdp)
print('Value for each property:', [round(d * 100, 2) for d in values.values()])
print('Error for each property:', [round(d * 100, 2) for d in diff.values()])
input_alphabet,
mdp_sul,
eq_oracle,
automaton_type='smm',
n_c=n_c,
n_resample=n_resample,
min_rounds=min_rounds,
strategy=strat,
max_rounds=max_rounds,
return_data=True,
samples_cex_strategy=cex_stat,
print_level=1,
cex_processing=cex_proc,
property_based_stopping=stopping_data)
smm_2_mdp = smm_to_mdp_conversion(learned_smm)
mdp_results, mdp_err = model_check_experiment(
get_properties_file(exp_name),
get_correct_prop_values(exp_name), learned_mdp)
smm_results, smm_err = model_check_experiment(
get_properties_file(exp_name),
get_correct_prop_values(exp_name), smm_2_mdp)
properties_string_header = ",".join(
[f'{key}_val,{key}_err' for key in mdp_results.keys()])
property_string_mdp = ",".join([
f'{str(mdp_results[p])},{str(mdp_err[p])}'
for p in mdp_results.keys()
])
