def __init__(self, path, reset_cost, default_reward, value_expert):
self.world = Model(path, reset_cost, default_reward)
self.value_expert = value_expert
in_letters = [Letter(symbol) for symbol in self.world.mrm.observations]
self.kbase = MRMActiveKnowledgeBase(self.world)
self.OT = ObservationTable(input_letters=in_letters,
knowledge_base=self.kbase)
print('Initializing OT')
self.OT.initialize()
print('OT initialized')
#COUNTERS
self.total_learning_time = 0
self.total_exploring_time = 0
self.rewards = 0
self.iteration4Explor = 0
self.iteration4OT = 0
self.nuof_counter_examples = 0
#EXECUTION
self.learn()
while not self.check():
self.learn()
#END PRINT
self.endPrints()
remove(TMP_MODEL_PATH)
def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None):
"""Implementation of the LSTAR algorithm.
Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod
by specifying the following 'eqtests' parameter:
eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7)
"""
self.input_letters = [Letter(symbol) for symbol in input_vocabulary]
self.knowledge_base = knowledge_base
self.tmp_dir = tmp_dir
self.observation_table = ObservationTable(self.input_letters, self.knowledge_base)
self.max_states = max_states
self.eqtests = eqtests
self.__f_stop = False
def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None):
"""Implementation of the LSTAR algorithm.
Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod
by specifying the following 'eqtests' parameter:
eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7)
"""
self.input_letters = [Letter(symbol) for symbol in input_vocabulary]
self.knowledge_base = knowledge_base
self.tmp_dir = tmp_dir
self.observation_table = ObservationTable(self.input_letters, self.knowledge_base)
self.max_states = max_states
self.eqtests = eqtests
class LSTAR(object):
"""TODO : Describe here the inner working of the LSTAR Algorithm
>>> from pylstar import LSTAR
>>> from pylstar import State
>>> from pylstar import Transition
>>> from pylstar import Automata
>>> from pylstar import Letter
>>> from pylstar import FakeActiveKnowledgeBase
>>> symbol_a = "a"
>>> symbol_b = "b"
>>> symbol_c = "c"
>>> symbol_1 = 1
>>> symbol_2 = 2
>>> symbol_3 = 3
>>> l_a = Letter(symbol_a)
>>> l_b = Letter(symbol_b)
>>> l_c = Letter(symbol_c)
>>> l_1 = Letter(symbol_1)
>>> l_2 = Letter(symbol_2)
>>> l_3 = Letter(symbol_3)
>>> s0 = State("S0")
>>> s1 = State("S1")
>>> s2 = State("S2")
>>> t1 = Transition("t1", output_state=s0, input_letter=l_a, output_letter=l_1)
>>> t2 = Transition("t2", output_state=s1, input_letter=l_b, output_letter=l_2)
>>> t3 = Transition("t3", output_state=s2, input_letter=l_c, output_letter=l_3)
>>> s0.transitions = [t1, t2, t3]
>>> t4 = Transition("t4", output_state=s1, input_letter=l_a, output_letter=l_2)
>>> t5 = Transition("t5", output_state=s1, input_letter=l_b, output_letter=l_3)
>>> t6 = Transition("t6", output_state=s0, input_letter=l_c, output_letter=l_1)
>>> s1.transitions = [t4, t5, t6]
>>> t7 = Transition("t7", output_state=s2, input_letter=l_a, output_letter=l_2)
>>> t8 = Transition("t8", output_state=s2, input_letter=l_b, output_letter=l_3)
>>> t9 = Transition("t9", output_state=s1, input_letter=l_c, output_letter=l_1)
>>> s2.transitions = [t7, t8, t9]
>>> automata = Automata(s0)
>>> kbase = FakeActiveKnowledgeBase(automata)
>>> input_vocabulary = [symbol_a, symbol_b, symbol_c]
>>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5)
>>> infered_automata = lstar.learn()
>>> print(infered_automata.build_dot_code())
digraph "Automata" {
"0" [shape=doubleoctagon, style=filled, fillcolor=white, URL="0"];
"2" [shape=ellipse, style=filled, fillcolor=white, URL="2"];
"1" [shape=ellipse, style=filled, fillcolor=white, URL="1"];
"0" -> "0" [fontsize=5, label="a / 1", URL="t0"];
"0" -> "1" [fontsize=5, label="b / 2", URL="t1"];
"0" -> "2" [fontsize=5, label="c / 3", URL="t2"];
"2" -> "2" [fontsize=5, label="a / 2", URL="t6"];
"2" -> "2" [fontsize=5, label="b / 3", URL="t7"];
"2" -> "1" [fontsize=5, label="c / 1", URL="t8"];
"1" -> "1" [fontsize=5, label="a / 2", URL="t3"];
"1" -> "1" [fontsize=5, label="b / 3", URL="t4"];
"1" -> "0" [fontsize=5, label="c / 1", URL="t5"];
}
>>> from pylstar import LSTAR
>>> from pylstar import State
>>> from pylstar import Transition
>>> from pylstar import Automata
>>> from pylstar import Letter
>>> from pylstar import FakeActiveKnowledgeBase
>>> # input symbols
>>> symbol_hello = "hello"
>>> symbol_bye = "bye"
>>> symbol_pass_valid = "pass valid"
>>> symbol_pass_invalid = "pass invalid"
>>> symbol_cmd1 = "cmd1"
>>> symbol_cmd2 = "cmd2"
>>> # output symbols
>>> symbol_pass_request = "pass?"
>>> symbol_ack = "ack"
>>> symbol_welcome = "welcome"
>>> symbol_error = "error"
>>> # create a letter for each symbol
>>> l_hello = Letter(symbol_hello)
>>> l_bye = Letter(symbol_bye)
>>> l_pass_valid = Letter(symbol_pass_valid)
>>> l_pass_invalid = Letter("pass invalid")
>>> l_cmd1 = Letter(symbol_cmd1)
>>> l_cmd2 = Letter(symbol_cmd2)
>>> l_welcome = Letter(symbol_welcome)
>>> l_ack = Letter(symbol_ack)
>>> l_pass_request = Letter(symbol_pass_request)
>>> l_error = Letter(symbol_error)
>>> # create the infered automata
>>> s0 = State("S0")
>>> s1 = State("S1")
>>> s2 = State("S2")
>>> t1 = Transition("t1", output_state=s1, input_letter=l_hello, output_letter=l_pass_request)
>>> t2 = Transition("t2", output_state=s0, input_letter=l_bye, output_letter=l_ack)
>>> t3 = Transition("t3", output_state=s0, input_letter=l_pass_valid, output_letter=l_error)
>>> t4 = Transition("t4", output_state=s0, input_letter=l_pass_invalid, output_letter=l_error)
>>> t5 = Transition("t5", output_state=s0, input_letter=l_cmd1, output_letter=l_error)
>>> t6 = Transition("t6", output_state=s0, input_letter=l_cmd2, output_letter=l_error)
>>> s0.transitions = [t1, t2, t3, t4, t5, t6]
>>> t7 = Transition("t7", output_state=s1, input_letter=l_hello, output_letter=l_error)
>>> t8 = Transition("t8", output_state=s0, input_letter=l_bye, output_letter=l_ack)
>>> t9 = Transition("t9", output_state=s2, input_letter=l_pass_valid, output_letter=l_welcome)
>>> t10 = Transition("t10", output_state=s1, input_letter=l_pass_invalid, output_letter=l_error)
>>> t11 = Transition("t11", output_state=s1, input_letter=l_cmd1, output_letter=l_error)
>>> t12 = Transition("t12", output_state=s1, input_letter=l_cmd2, output_letter=l_error)
>>> s1.transitions = [t7, t8, t9, t10, t11, t12]
>>> t13 = Transition("t13", output_state=s2, input_letter=l_hello, output_letter=l_error)
>>> t14 = Transition("t14", output_state=s0, input_letter=l_bye, output_letter=l_ack)
>>> t15 = Transition("t15", output_state=s2, input_letter=l_pass_valid, output_letter=l_error)
>>> t16 = Transition("t16", output_state=s2, input_letter=l_pass_invalid, output_letter=l_error)
>>> t17 = Transition("t17", output_state=s2, input_letter=l_cmd1, output_letter=l_ack)
>>> t18 = Transition("t18", output_state=s2, input_letter=l_cmd2, output_letter=l_ack)
>>> s2.transitions = [t13, t14, t15, t16, t17, t18]
>>> automata = Automata(s0)
>>> kbase = FakeActiveKnowledgeBase(automata)
>>> input_vocabulary = [symbol_hello, symbol_bye, symbol_pass_valid, symbol_pass_invalid, symbol_cmd1, symbol_cmd2]
>>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5)
>>> infered_automata = lstar.learn()
>>> print(infered_automata.build_dot_code())
digraph "Automata" {
"0" [shape=doubleoctagon, style=filled, fillcolor=white, URL="0"];
"1" [shape=ellipse, style=filled, fillcolor=white, URL="1"];
"2" [shape=ellipse, style=filled, fillcolor=white, URL="2"];
"0" -> "1" [fontsize=5, label="hello / pass?", URL="t0"];
"0" -> "0" [fontsize=5, label="bye / ack", URL="t1"];
"0" -> "0" [fontsize=5, label="pass valid / error", URL="t2"];
"0" -> "0" [fontsize=5, label="pass invalid / error", URL="t3"];
"0" -> "0" [fontsize=5, label="cmd1 / error", URL="t4"];
"0" -> "0" [fontsize=5, label="cmd2 / error", URL="t5"];
"1" -> "1" [fontsize=5, label="hello / error", URL="t6"];
"1" -> "0" [fontsize=5, label="bye / ack", URL="t7"];
"1" -> "2" [fontsize=5, label="pass valid / welcome", URL="t8"];
"1" -> "1" [fontsize=5, label="pass invalid / error", URL="t9"];
"1" -> "1" [fontsize=5, label="cmd1 / error", URL="t10"];
"1" -> "1" [fontsize=5, label="cmd2 / error", URL="t11"];
"2" -> "2" [fontsize=5, label="hello / error", URL="t12"];
"2" -> "0" [fontsize=5, label="bye / ack", URL="t13"];
"2" -> "2" [fontsize=5, label="pass valid / error", URL="t14"];
"2" -> "2" [fontsize=5, label="pass invalid / error", URL="t15"];
"2" -> "2" [fontsize=5, label="cmd1 / ack", URL="t16"];
"2" -> "2" [fontsize=5, label="cmd2 / ack", URL="t17"];
}
>>> from pylstar import LSTAR
>>> from pylstar import State
>>> from pylstar import Transition
>>> from pylstar import Automata
>>> from pylstar import Letter
>>> from pylstar import FakeActiveKnowledgeBase
>>> symbol_a = "a"
>>> symbol_b = "b"
>>> symbol_c = "c"
>>> symbol_1 = 1
>>> symbol_2 = 2
>>> symbol_3 = 3
>>> l_a = Letter(symbol_a)
>>> l_b = Letter(symbol_b)
>>> l_c = Letter(symbol_c)
>>> l_1 = Letter(symbol_1)
>>> l_2 = Letter(symbol_2)
>>> l_3 = Letter(symbol_3)
>>> s0 = State("S0")
>>> s1 = State("S1")
>>> s2 = State("S2")
>>> t1 = Transition("t1", output_state=s0, input_letter=l_a, output_letter=l_1)
>>> t2 = Transition("t2", output_state=s1, input_letter=l_b, output_letter=l_2)
>>> t3 = Transition("t3", output_state=s2, input_letter=l_c, output_letter=l_3)
>>> s0.transitions = [t1, t2, t3]
>>> t4 = Transition("t4", output_state=s2, input_letter=l_a, output_letter=l_3)
>>> t5 = Transition("t5", output_state=s0, input_letter=l_b, output_letter=l_1)
>>> t6 = Transition("t6", output_state=s1, input_letter=l_c, output_letter=l_2)
>>> s1.transitions = [t4, t5, t6]
>>> t7 = Transition("t7", output_state=s1, input_letter=l_a, output_letter=l_2)
>>> t8 = Transition("t8", output_state=s2, input_letter=l_b, output_letter=l_3)
>>> t9 = Transition("t9", output_state=s0, input_letter=l_c, output_letter=l_1)
>>> s2.transitions = [t7, t8, t9]
>>> automata = Automata(s0)
>>> kbase = FakeActiveKnowledgeBase(automata)
>>> input_vocabulary = [symbol_a, symbol_b, symbol_c]
>>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5)
>>> infered_automata = lstar.learn()
>>> print(infered_automata.build_dot_code())
digraph "Automata" {
"0" [shape=doubleoctagon, style=filled, fillcolor=white, URL="0"];
"2" [shape=ellipse, style=filled, fillcolor=white, URL="2"];
"1" [shape=ellipse, style=filled, fillcolor=white, URL="1"];
"0" -> "0" [fontsize=5, label="a / 1", URL="t0"];
"0" -> "1" [fontsize=5, label="b / 2", URL="t1"];
"0" -> "2" [fontsize=5, label="c / 3", URL="t2"];
"2" -> "1" [fontsize=5, label="a / 2", URL="t6"];
"2" -> "2" [fontsize=5, label="b / 3", URL="t7"];
"2" -> "0" [fontsize=5, label="c / 1", URL="t8"];
"1" -> "2" [fontsize=5, label="a / 3", URL="t3"];
"1" -> "0" [fontsize=5, label="b / 1", URL="t4"];
"1" -> "1" [fontsize=5, label="c / 2", URL="t5"];
}
"""
def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None):
"""Implementation of the LSTAR algorithm.
Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod
by specifying the following 'eqtests' parameter:
eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7)
"""
self.input_letters = [Letter(symbol) for symbol in input_vocabulary]
self.knowledge_base = knowledge_base
self.tmp_dir = tmp_dir
self.observation_table = ObservationTable(self.input_letters, self.knowledge_base)
self.max_states = max_states
self.eqtests = eqtests
self.__f_stop = False
def stop(self):
"""This method can be use to trigger the end of the learning process"""
self._logger.info("Stopping the LSTAR learning process.")
self.__f_stop = True
def learn(self):
self._logger.info("Starting the LSTAR learning process.")
# intialization
self.__initialize()
f_hypothesis_is_valid = False
i_round = 1
while not f_hypothesis_is_valid and not self.__f_stop:
hypothesis = self.build_hypothesis(i_round)
self.__serialize_hypothesis(i_round, hypothesis)
counterexample = self.eqtests.find_counterexample(hypothesis)
if counterexample is not None:
self._logger.info("Counterexample '{}' found.".format(counterexample))
self.fix_hypothesis(counterexample)
else:
f_hypothesis_is_valid = True
i_round += 1
self.__serialize_observation_table(i_round)
self._logger.info("Automata successfully computed")
return hypothesis
def __serialize_hypothesis(self, i_round, hypothesis):
if i_round is None:
raise Exception("i_round cannot be None")
if hypothesis is None:
raise Exception("Hypothesis cannot be None")
dot_code = hypothesis.build_dot_code()
filepath = os.path.join(self.tmp_dir, "hypothesis_{}.dot".format(i_round))
with open(filepath, 'w') as fd:
fd.write(dot_code)
self._logger.info("Hypothesis produced on round '{}' stored in '{}'".format(i_round, filepath))
def __serialize_observation_table(self, i_round):
if self.observation_table is None:
raise Exception("Observation table cannot ne Bone")
serialized_table = self.observation_table.serialize()
str_date = datetime.strftime(datetime.now(), "%Y%m%d_%H%M%S")
filepath = os.path.join(self.tmp_dir, "observation_table_{}_{}.raw".format(i_round, str_date))
with open(filepath, 'w') as fd:
fd.write(serialized_table)
self._logger.info("Observation table serialized in '{}'".format(filepath))
def fix_hypothesis(self, counterexample):
if counterexample is None:
raise Exception("counterexample cannot be None")
self._logger.debug("fix hypothesis with counterexample '{}'".format(counterexample))
input_word = counterexample.input_word
output_word = counterexample.output_word
self.observation_table.add_counterexample(input_word, output_word)
def build_hypothesis(self, i_round):
if i_round is None:
raise Exception("i_round cannot be None")
f_consistent = False
f_closed = False
self._logger.info("Building the hypothesis ({} round)".format(i_round))
while not f_consistent or not f_closed:
if not self.observation_table.is_closed():
self._logger.info("Observation table is not closed.")
self.observation_table.close_table()
f_closed = False
else:
self._logger.info("Observation table is closed")
f_closed = True
inconsistency = self.observation_table.find_inconsistency()
if inconsistency is not None:
self._logger.info("Observation table is not consistent.")
self.observation_table.make_consistent(inconsistency)
f_consistent = False
else:
self._logger.info("Observation table is consistent")
f_consistent = True
self.__serialize_observation_table(i_round)
self._logger.info("Hypothesis computed")
return self.observation_table.build_hypothesis()
def __initialize(self):
"""Initialization of the observation table"""
self.observation_table.initialize()
self._logger.info("Observation table is initialized")
self._logger.info("\n"+str(self.observation_table))
@property
def input_vocabulary(self):
"""Input_vocabulary to use """
return self.__input_vocabulary
@input_vocabulary.setter
def input_vocabulary(self, input_vocabulary):
if input_vocabulary is None:
raise ValueError("Input_vocabulary cannot be None")
if len(input_vocabulary) == 0:
raise ValueError("Input vocabulary cannot be empty")
self.__input_vocabulary = input_vocabulary
@property
def knowledge_base(self):
"""Membership Knowledge_base"""
return self.__knowledge_base
@knowledge_base.setter
def knowledge_base(self, knowledge_base):
if knowledge_base is None:
raise ValueError("Knowledge_base cannot be None")
self.__knowledge_base = knowledge_base
@property
def tmp_dir(self):
"""Temporary directory that host serialized observation tables and hypothesis"""
return self.__tmp_dir
@tmp_dir.setter
def tmp_dir(self, value):
if value is None:
self.__tmp_dir = tempfile.mkdtemp(prefix='pylstar_')
else:
self.__tmp_dir = value
@property
def eqtests(self):
return self.__eqtests
@eqtests.setter
def eqtests(self, eqtests):
if eqtests is None:
self.__eqtests = WpMethodEQ(self.knowledge_base, self.max_states, self.input_letters)
else:
self.__eqtests = eqtests
class LSTAR(object):
"""TODO : Describe here the inner working of the LSTAR Algorithm
>>> from pylstar.LSTAR import LSTAR
>>> from pylstar.automata.State import State
>>> from pylstar.automata.Transition import Transition
>>> from pylstar.automata.Automata import Automata
>>> from pylstar.Letter import Letter
>>> from pylstar.FakeActiveKnowledgeBase import FakeActiveKnowledgeBase
>>> symbol_a = "a"
>>> symbol_b = "b"
>>> symbol_c = "c"
>>> symbol_1 = 1
>>> symbol_2 = 2
>>> symbol_3 = 3
>>> l_a = Letter(symbol_a)
>>> l_b = Letter(symbol_b)
>>> l_c = Letter(symbol_c)
>>> l_1 = Letter(symbol_1)
>>> l_2 = Letter(symbol_2)
>>> l_3 = Letter(symbol_3)
>>> s0 = State("S0")
>>> s1 = State("S1")
>>> s2 = State("S2")
>>> t1 = Transition("t1", output_state=s0, input_letter=l_a, output_letter=l_1)
>>> t2 = Transition("t2", output_state=s1, input_letter=l_b, output_letter=l_2)
>>> t3 = Transition("t3", output_state=s2, input_letter=l_c, output_letter=l_3)
>>> s0.transitions = [t1, t2, t3]
>>> t4 = Transition("t4", output_state=s1, input_letter=l_a, output_letter=l_2)
>>> t5 = Transition("t5", output_state=s1, input_letter=l_b, output_letter=l_3)
>>> t6 = Transition("t6", output_state=s0, input_letter=l_c, output_letter=l_1)
>>> s1.transitions = [t4, t5, t6]
>>> t7 = Transition("t7", output_state=s2, input_letter=l_a, output_letter=l_2)
>>> t8 = Transition("t8", output_state=s2, input_letter=l_b, output_letter=l_3)
>>> t9 = Transition("t9", output_state=s1, input_letter=l_c, output_letter=l_1)
>>> s2.transitions = [t7, t8, t9]
>>> automata = Automata(s0)
>>> kbase = FakeActiveKnowledgeBase(automata)
>>> input_vocabulary = [symbol_a, symbol_b, symbol_c]
>>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5)
>>> infered_automata = lstar.learn()
>>> print infered_automata.build_dot_code()
digraph G {
"1,2,3,2" [shape=doubleoctagon, style=filled, fillcolor=white, URL="1,2,3,2"];
"2,3,1,2" [shape=ellipse, style=filled, fillcolor=white, URL="2,3,1,2"];
"2,3,1,1" [shape=ellipse, style=filled, fillcolor=white, URL="2,3,1,1"];
"1,2,3,2" -> "1,2,3,2" [fontsize=5, label="I='Letter('a')' / O='Letter(1)'", URL="t0"];
"1,2,3,2" -> "2,3,1,1" [fontsize=5, label="I='Letter('b')' / O='Letter(2)'", URL="t1"];
"1,2,3,2" -> "2,3,1,2" [fontsize=5, label="I='Letter('c')' / O='Letter(3)'", URL="t2"];
"2,3,1,2" -> "2,3,1,2" [fontsize=5, label="I='Letter('a')' / O='Letter(2)'", URL="t6"];
"2,3,1,2" -> "2,3,1,2" [fontsize=5, label="I='Letter('b')' / O='Letter(3)'", URL="t7"];
"2,3,1,2" -> "2,3,1,1" [fontsize=5, label="I='Letter('c')' / O='Letter(1)'", URL="t8"];
"2,3,1,1" -> "2,3,1,1" [fontsize=5, label="I='Letter('a')' / O='Letter(2)'", URL="t3"];
"2,3,1,1" -> "2,3,1,1" [fontsize=5, label="I='Letter('b')' / O='Letter(3)'", URL="t4"];
"2,3,1,1" -> "1,2,3,2" [fontsize=5, label="I='Letter('c')' / O='Letter(1)'", URL="t5"];
}
>>> from pylstar.LSTAR import LSTAR
>>> from pylstar.automata.State import State
>>> from pylstar.automata.Transition import Transition
>>> from pylstar.automata.Automata import Automata
>>> from pylstar.Letter import Letter
>>> from pylstar.FakeActiveKnowledgeBase import FakeActiveKnowledgeBase
>>> # input symbols
>>> symbol_hello = "hello"
>>> symbol_bye = "bye"
>>> symbol_pass_valid = "pass valid"
>>> symbol_pass_invalid = "pass invalid"
>>> symbol_cmd1 = "cmd1"
>>> symbol_cmd2 = "cmd2"
>>> # output symbols
>>> symbol_pass_request = "pass?"
>>> symbol_ack = "ack"
>>> symbol_welcome = "welcome"
>>> symbol_error = "error"
>>> # create a letter for each symbol
>>> l_hello = Letter(symbol_hello)
>>> l_bye = Letter(symbol_bye)
>>> l_pass_valid = Letter(symbol_pass_valid)
>>> l_pass_invalid = Letter("pass invalid")
>>> l_cmd1 = Letter(symbol_cmd1)
>>> l_cmd2 = Letter(symbol_cmd2)
>>> l_welcome = Letter(symbol_welcome)
>>> l_ack = Letter(symbol_ack)
>>> l_pass_request = Letter(symbol_pass_request)
>>> l_error = Letter(symbol_error)
>>> # create the infered automata
>>> s0 = State("S0")
>>> s1 = State("S1")
>>> s2 = State("S2")
>>> t1 = Transition("t1", output_state=s1, input_letter=l_hello, output_letter=l_pass_request)
>>> t2 = Transition("t2", output_state=s0, input_letter=l_bye, output_letter=l_ack)
>>> t3 = Transition("t3", output_state=s0, input_letter=l_pass_valid, output_letter=l_error)
>>> t4 = Transition("t4", output_state=s0, input_letter=l_pass_invalid, output_letter=l_error)
>>> t5 = Transition("t5", output_state=s0, input_letter=l_cmd1, output_letter=l_error)
>>> t6 = Transition("t6", output_state=s0, input_letter=l_cmd2, output_letter=l_error)
>>> s0.transitions = [t1, t2, t3, t4, t5, t6]
>>> t7 = Transition("t7", output_state=s1, input_letter=l_hello, output_letter=l_error)
>>> t8 = Transition("t8", output_state=s0, input_letter=l_bye, output_letter=l_ack)
>>> t9 = Transition("t9", output_state=s2, input_letter=l_pass_valid, output_letter=l_welcome)
>>> t10 = Transition("t10", output_state=s1, input_letter=l_pass_invalid, output_letter=l_error)
>>> t11 = Transition("t11", output_state=s1, input_letter=l_cmd1, output_letter=l_error)
>>> t12 = Transition("t12", output_state=s1, input_letter=l_cmd2, output_letter=l_error)
>>> s1.transitions = [t7, t8, t9, t10, t11, t12]
>>> t13 = Transition("t13", output_state=s2, input_letter=l_hello, output_letter=l_error)
>>> t14 = Transition("t14", output_state=s0, input_letter=l_bye, output_letter=l_ack)
>>> t15 = Transition("t15", output_state=s2, input_letter=l_pass_valid, output_letter=l_error)
>>> t16 = Transition("t16", output_state=s2, input_letter=l_pass_invalid, output_letter=l_error)
>>> t17 = Transition("t17", output_state=s2, input_letter=l_cmd1, output_letter=l_ack)
>>> t18 = Transition("t18", output_state=s2, input_letter=l_cmd2, output_letter=l_ack)
>>> s2.transitions = [t13, t14, t15, t16, t17, t18]
>>> automata = Automata(s0)
>>> kbase = FakeActiveKnowledgeBase(automata)
>>> input_vocabulary = [symbol_hello, symbol_bye, symbol_pass_valid, symbol_pass_invalid, symbol_cmd1, symbol_cmd2]
>>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5)
>>> infered_automata = lstar.learn()
>>> print infered_automata.build_dot_code()
digraph G {
"'pass?','ack','error','error','error','error'" [shape=doubleoctagon, style=filled, fillcolor=white, URL="'pass?','ack','error','error','error','error'"];
"'error','ack','welcome','error','error','error'" [shape=ellipse, style=filled, fillcolor=white, URL="'error','ack','welcome','error','error','error'"];
"'error','ack','error','error','ack','ack'" [shape=ellipse, style=filled, fillcolor=white, URL="'error','ack','error','error','ack','ack'"];
"'pass?','ack','error','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('hello')' / O='Letter('pass?')'", URL="t0"];
"'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('bye')' / O='Letter('ack')'", URL="t1"];
"'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('pass valid')' / O='Letter('error')'", URL="t2"];
"'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('pass invalid')' / O='Letter('error')'", URL="t3"];
"'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('cmd1')' / O='Letter('error')'", URL="t4"];
"'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('cmd2')' / O='Letter('error')'", URL="t5"];
"'error','ack','welcome','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('hello')' / O='Letter('error')'", URL="t6"];
"'error','ack','welcome','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('bye')' / O='Letter('ack')'", URL="t7"];
"'error','ack','welcome','error','error','error'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('pass valid')' / O='Letter('welcome')'", URL="t8"];
"'error','ack','welcome','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('pass invalid')' / O='Letter('error')'", URL="t9"];
"'error','ack','welcome','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('cmd1')' / O='Letter('error')'", URL="t10"];
"'error','ack','welcome','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('cmd2')' / O='Letter('error')'", URL="t11"];
"'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('hello')' / O='Letter('error')'", URL="t12"];
"'error','ack','error','error','ack','ack'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('bye')' / O='Letter('ack')'", URL="t13"];
"'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('pass valid')' / O='Letter('error')'", URL="t14"];
"'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('pass invalid')' / O='Letter('error')'", URL="t15"];
"'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('cmd1')' / O='Letter('ack')'", URL="t16"];
"'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('cmd2')' / O='Letter('ack')'", URL="t17"];
}
>>> from pylstar.LSTAR import LSTAR
>>> from pylstar.automata.State import State
>>> from pylstar.automata.Transition import Transition
>>> from pylstar.automata.Automata import Automata
>>> from pylstar.Letter import Letter
>>> from pylstar.FakeActiveKnowledgeBase import FakeActiveKnowledgeBase
>>> symbol_a = "a"
>>> symbol_b = "b"
>>> symbol_c = "c"
>>> symbol_1 = 1
>>> symbol_2 = 2
>>> symbol_3 = 3
>>> l_a = Letter(symbol_a)
>>> l_b = Letter(symbol_b)
>>> l_c = Letter(symbol_c)
>>> l_1 = Letter(symbol_1)
>>> l_2 = Letter(symbol_2)
>>> l_3 = Letter(symbol_3)
>>> s0 = State("S0")
>>> s1 = State("S1")
>>> s2 = State("S2")
>>> t1 = Transition("t1", output_state=s0, input_letter=l_a, output_letter=l_1)
>>> t2 = Transition("t2", output_state=s1, input_letter=l_b, output_letter=l_2)
>>> t3 = Transition("t3", output_state=s2, input_letter=l_c, output_letter=l_3)
>>> s0.transitions = [t1, t2, t3]
>>> t4 = Transition("t4", output_state=s2, input_letter=l_a, output_letter=l_3)
>>> t5 = Transition("t5", output_state=s0, input_letter=l_b, output_letter=l_1)
>>> t6 = Transition("t6", output_state=s1, input_letter=l_c, output_letter=l_2)
>>> s1.transitions = [t4, t5, t6]
>>> t7 = Transition("t7", output_state=s1, input_letter=l_a, output_letter=l_2)
>>> t8 = Transition("t8", output_state=s2, input_letter=l_b, output_letter=l_3)
>>> t9 = Transition("t9", output_state=s0, input_letter=l_c, output_letter=l_1)
>>> s2.transitions = [t7, t8, t9]
>>> automata = Automata(s0)
>>> kbase = FakeActiveKnowledgeBase(automata)
>>> input_vocabulary = [symbol_a, symbol_b, symbol_c]
>>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5)
>>> infered_automata = lstar.learn()
>>> print infered_automata.build_dot_code()
digraph G {
"1,2,3" [shape=doubleoctagon, style=filled, fillcolor=white, URL="1,2,3"];
"2,3,1" [shape=ellipse, style=filled, fillcolor=white, URL="2,3,1"];
"3,1,2" [shape=ellipse, style=filled, fillcolor=white, URL="3,1,2"];
"1,2,3" -> "1,2,3" [fontsize=5, label="I='Letter('a')' / O='Letter(1)'", URL="t6"];
"1,2,3" -> "3,1,2" [fontsize=5, label="I='Letter('b')' / O='Letter(2)'", URL="t7"];
"1,2,3" -> "2,3,1" [fontsize=5, label="I='Letter('c')' / O='Letter(3)'", URL="t8"];
"2,3,1" -> "3,1,2" [fontsize=5, label="I='Letter('a')' / O='Letter(2)'", URL="t3"];
"2,3,1" -> "2,3,1" [fontsize=5, label="I='Letter('b')' / O='Letter(3)'", URL="t4"];
"2,3,1" -> "1,2,3" [fontsize=5, label="I='Letter('c')' / O='Letter(1)'", URL="t5"];
"3,1,2" -> "2,3,1" [fontsize=5, label="I='Letter('a')' / O='Letter(3)'", URL="t0"];
"3,1,2" -> "1,2,3" [fontsize=5, label="I='Letter('b')' / O='Letter(1)'", URL="t1"];
"3,1,2" -> "3,1,2" [fontsize=5, label="I='Letter('c')' / O='Letter(2)'", URL="t2"];
}
"""
def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None):
"""Implementation of the LSTAR algorithm.
Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod
by specifying the following 'eqtests' parameter:
eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7)
"""
self.input_letters = [Letter(symbol) for symbol in input_vocabulary]
self.knowledge_base = knowledge_base
self.tmp_dir = tmp_dir
self.observation_table = ObservationTable(self.input_letters, self.knowledge_base)
self.max_states = max_states
self.eqtests = eqtests
def learn(self):
self._logger.info("Starting the LSTAR learning process.")
# intialization
self.__initialize()
f_hypothesis_is_valid = False
i_round = 1
while not f_hypothesis_is_valid:
hypothesis = self.build_hypothesis(i_round)
self.__serialize_hypothesis(i_round, hypothesis)
counterexample = self.eqtests.find_counterexample(hypothesis)
if counterexample is not None:
self._logger.info("Counterexample '{}' found.".format(counterexample))
self.fix_hypothesis(counterexample)
else:
f_hypothesis_is_valid = True
i_round += 1
self._logger.info("Automata successfully computed")
return hypothesis
def __serialize_hypothesis(self, i_round, hypothesis):
if i_round is None:
raise Exception("i_round cannot be None")
if hypothesis is None:
raise Exception("Hypothesis cannot be None")
dot_code = hypothesis.build_dot_code()
filepath = os.path.join(self.tmp_dir, "hypothesis_{}.dot".format(i_round))
with open(filepath, 'w') as fd:
fd.write(dot_code)
self._logger.info("Hypothesis produced on round '{}' stored in '{}'".format(i_round, filepath))
def __serialize_observation_table(self, i_round):
if self.observation_table is None:
raise Exception("Observation table cannot ne Bone")
serialized_table = self.observation_table.serialize()
str_date = datetime.strftime(datetime.now(), "%Y%m%d_%H%M%S")
filepath = os.path.join(self.tmp_dir, "observation_table_{}_{}.raw".format(i_round, str_date))
with open(filepath, 'w') as fd:
fd.write(serialized_table)
self._logger.info("Observation table serialized in '{}'".format(filepath))
def fix_hypothesis(self, counterexample):
if counterexample is None:
raise Exception("counterexample cannot be None")
self._logger.debug("fix hypothesis with counterexample '{}'".format(counterexample))
input_word = counterexample.input_word
output_word = counterexample.output_word
self.observation_table.add_counterexample(input_word, output_word)
def build_hypothesis(self, i_round):
if i_round is None:
raise Exception("i_round cannot be None")
f_consistent = False
f_closed = False
while not f_consistent or not f_closed:
if not self.observation_table.is_closed():
self._logger.info("Observation table is not closed.")
self.observation_table.close_table()
f_closed = False
else:
self._logger.info("Observation table is closed")
f_closed = True
inconsistency = self.observation_table.find_inconsistency()
if inconsistency is not None:
self._logger.info("Observation table is not consistent.")
self.observation_table.make_consistent(inconsistency)
f_consistent = False
else:
self._logger.info("Observation table is consistent")
f_consistent = True
self.__serialize_observation_table(i_round)
self._logger.info("Hypothesis computed")
return self.observation_table.build_hypothesis()
def __initialize(self):
"""Initialization of the observation table"""
self.observation_table.initialize()
self._logger.info("Observation table is initialized")
self._logger.info("\n"+str(self.observation_table))
@property
def input_vocabulary(self):
"""Input_vocabulary to use """
return self.__input_vocabulary
@input_vocabulary.setter
def input_vocabulary(self, input_vocabulary):
if input_vocabulary is None:
raise ValueError("Input_vocabulary cannot be None")
if len(input_vocabulary) == 0:
raise ValueError("Input vocabulary cannot be empty")
self.__input_vocabulary = input_vocabulary
@property
def knowledge_base(self):
"""Membership Knowledge_base"""
return self.__knowledge_base
@knowledge_base.setter
def knowledge_base(self, knowledge_base):
if knowledge_base is None:
raise ValueError("Knowledge_base cannot be None")
self.__knowledge_base = knowledge_base
@property
def tmp_dir(self):
"""Temporary directory that host serialized observation tables and hypothesis"""
return self.__tmp_dir
@tmp_dir.setter
def tmp_dir(self, value):
if value is None:
self.__tmp_dir = tempfile.mkdtemp(prefix='pylstar_')
else:
self.__tmp_dir = value
@property
def eqtests(self):
return self.__eqtests
@eqtests.setter
def eqtests(self, eqtests):
if eqtests is None:
self.__eqtests = WpMethodEQ(self.knowledge_base, self.max_states, self.input_letters)
else:
self.__eqtests = eqtests
class LearningMRM:
"""Main class of the framework"""
def __init__(self, path, reset_cost, default_reward, value_expert):
self.world = Model(path, reset_cost, default_reward)
self.value_expert = value_expert
in_letters = [Letter(symbol) for symbol in self.world.mrm.observations]
self.kbase = MRMActiveKnowledgeBase(self.world)
self.OT = ObservationTable(input_letters=in_letters,
knowledge_base=self.kbase)
print('Initializing OT')
self.OT.initialize()
print('OT initialized')
#COUNTERS
self.total_learning_time = 0
self.total_exploring_time = 0
self.rewards = 0
self.iteration4Explor = 0
self.iteration4OT = 0
self.nuof_counter_examples = 0
#EXECUTION
self.learn()
while not self.check():
self.learn()
#END PRINT
self.endPrints()
remove(TMP_MODEL_PATH)
def learn(self):
"""Use L*_M algorithm to learn the MRM"""
StartTime = time.time()
iteration4OT = 0
closed = self.OT.is_closed()
inconsistency = self.OT.find_inconsistency()
while not closed or inconsistency is not None:
iteration4OT += 1
print('Building the OT;', 'iteration', iteration4OT)
if not closed:
self.OT.close_table()
if inconsistency is not None:
self.OT.make_consistent(inconsistency)
closed = self.OT.is_closed()
inconsistency = self.OT.find_inconsistency()
EndTime = time.time()
self.total_learning_time += EndTime - StartTime
def check(self):
"""Check if the hypothesis is correct"""
StartTime = time.time()
self.createHypothesis()
if not self.passedFirstCheck(): # Expert value check
res = False
elif not self.passedSecondCheck(): # Exploitation check
res = False
else:
res = True
EndTime = time.time()
self.total_exploring_time += EndTime - StartTime
return res
def endPrints(self):
print()
print('Optimization problem:', ["MIN", "MAX"][MODE])
print('# learning actions:', self.kbase.actionsForLearning)
print()
print('rewards:', self.rewards)
print('nuof_MQs:', self.kbase.nuof_MQs)
print('Exploration iterations:', self.iteration4Explor)
print('nuof_counter_examples:', self.nuof_counter_examples)
print('total_learning_time:', self.total_learning_time)
print('total_exploring_time:', self.total_exploring_time)
def createHypothesis(self):
print()
print('Building hypothesis MRM...')
RM = self.OT.build_hypothesis()
print('Hypothesis MRM built !')
self.buildProductAutomaton(
RM) # Write the prism file with the hypothesis
program = stormpy.parse_prism_program(TMP_MODEL_PATH)
properties = stormpy.parse_properties_for_prism_program(
"Rmax=? [ LRA ]", program)
options = stormpy.BuilderOptions(True,
True) #To keep rewards and labels
self.h = stormpy.build_sparse_model_with_options(program, options)
self.result_h = stormpy.model_checking(self.h,
properties[0],
extract_scheduler=True).at(0)
self.scheduler = stormpy.model_checking(
self.h, properties[0], extract_scheduler=True).scheduler
def passedFirstCheck(self):
# Expert value check
if self.result_h < self.value_expert:
print(self.result_h, " < Value_expert:", self.value_expert,
", looking for counter-example...")
self.findCounterExample()
return False
else:
print(self.result_h, " >= Value_expert:", self.value_expert)
return True
def passedSecondCheck(self):
# Exploitation check. Check if the hypothesis is correct by executing the strategy
# After STERPS_PER_EPISODE we reset the system
# We end if we observe a counter example or if we have already performed ACTIONTOTEXECUTE actions
actionsExecuted = 0
while actionsExecuted < ACTIONTOEXECUTE:
self.iteration4Explor += 1
print()
print('Exploration iteration', self.iteration4Explor)
# Initialize agent's state to a rand position and
current_state_h = self.resetExploration()
for step in range(STEPS_PER_EPISODE):
a = self.scheduler.get_choice(current_state_h)
if self.isResetAction(a):
next_state_h = self.resetExploration()
obs = "null"
r_h = self.world.mrm.reset_cost
r_m = self.world.mrm.reset_cost
next_state_m = self.world.map.current
else:
(r_h, r_m, next_state_h, next_state_m,
obs) = self.executeOneStepExploration(
current_state_h, a.get_deterministic_choice())
self.updateTraces(obs, r_m)
actionsExecuted += 1
if self.isCounterExample(r_h, r_m):
self.nuof_counter_examples += 1
return False
else:
current_state_h = next_state_h
self.world.map.current = next_state_m
return True
def getStateInHypothesis(self, states_h, state):
for i in states_h:
if int(i.name) == int(state):
return i
def buildProductAutomaton(self, h):
"""Given a hypothesis of the angluin algo, build the product between the gird and this hypothesis and write it in a PRISM file.
The init state is {'c1','r1','null'} with no obs already made"""
rewards = "rewards\n"
labels = ''
out_file = open(TMP_MODEL_PATH, 'w')
#module
out_file.write("mdp\n\nmodule tmp\n\n")
#number of state and initial state
new_states = []
for s in self.world.map.states:
for o in range(len(h.get_states())):
labels += 'label "' + s + '_' + str(o) + '" = s=' + str(
len(new_states)) + ' ;\n'
new_states.append((s, o))
out_file.write("\ts : [0.." + str(len(new_states) - 1) + "] init " +
str(new_states.index((self.world.map.initiales[0],
0))) + ";\n\n")
#transitions
for s in new_states:
state_id = self.world.map.getIdState(s[0])
for a in self.world.map.availableActions(s[0]):
action_id = self.world.map.getIdAction(a)
obs = self.world.map.labelling[state_id][action_id]
#if len(self.world.map.transitions[state_id][action_id]) > 0:
out_file.write("\t[" + a + "] s=" + str(new_states.index(s)) +
"-> ")
temp_list = []
if obs == 'null':
rewards += "\t[" + a + "] (s=" + str(
new_states.index(s)) + ") : " + str(
self.world.mrm.default_reward) + ";\n"
for [dest, prob
] in self.world.map.transitions[state_id][action_id]:
index_dest = str(
new_states.index(
(self.world.map.getStateFromId(dest), s[1])))
temp_list.append(
str(prob) + " : (s'=" + index_dest + ")")
else:
tr_val = h.play_word(
Word([Letter(obs)]),
self.getStateInHypothesis(h.get_states(), s[1]))
state_in_h = int(tr_val[1][-1].name)
rewards += "\t[" + a + "] (s=" + str(
new_states.index(s)) + ") : " + str(
tr_val[0].last_letter().name) + ";\n"
for [dest, prob
] in self.world.map.transitions[state_id][action_id]:
index_dest = str(
new_states.index(
(self.world.map.getStateFromId(dest),
state_in_h)))
temp_list.append(
str(prob) + " : (s'=" + index_dest + ")")
out_file.write(" + ".join(temp_list))
out_file.write(";\n")
a = "reset"
out_file.write(
"\t[" + a + "] s=" + str(new_states.index(s)) +
"-> 1.0 : (s'=" +
str(new_states.index((self.world.map.initiales[0], 0))) +
");\n")
rewards += "\t[" + a + "] (s=" + str(
new_states.index(s)) + ") : " + str(
self.world.mrm.reset_cost) + ";\n"
out_file.write("\nendmodule\n\n")
out_file.write(labels)
rewards += "endrewards\n"
out_file.write(rewards)
out_file.close()
def resetH(self):
for s in range(len(self.h.states)):
if {str(self.world.map.current) + '_0'
}.issubset(self.h.states[s].labels):
return s
def getNextSateH(self, state, action):
action = state.actions[action]
r = random.random()
c = 0
for transition in action.transitions:
c += transition.value()
if r < c:
break
return transition.column
def getRewardH(self, state, action):
c = 0
for i in range(state): # i: id in h => state in h => state in m
c += len(self.h.states[i].actions)
return self.h.reward_models[''].state_action_rewards[c + int(
action.__str__())] # +1 because we have the reset action
def fromIdStateHToIdStateM(self, sh):
pattern = re.compile("s[0-9]+_[0-9]+")
for i in self.h.states[sh].labels:
if pattern.match(i):
return i[:i.index('_')]
def executeOneStepExploration(self, current_state_h, action):
next_state_h = self.getNextSateH(self.h.states[current_state_h],
action)
r_h = self.getRewardH(current_state_h, action)
obs = self.world.map.labelling[self.world.map.getIdState(
self.world.map.current)][int(action.__str__())]
next_state_m = self.fromIdStateHToIdStateM(next_state_h)
#[next_state_m,obs] = self.world.map.moveFrom(self.world.map.current,self.world.map.actions[int(action.__str__())])
r_m = None
if obs == 'null':
r_m = self.world.mrm.default_reward
else:
r_m = self.world.mrm.move(obs)
return (r_h, r_m, next_state_h, next_state_m, obs)
def isCounterExample(self, r_h, r_m):
"""Return True if the two rewards r_h and r_m are different and add the counter example at the OT."""
if r_m != r_h:
print("CE", r_m, r_h, self.observation_seq)
input_word = Word(
[Letter(symbol) for symbol in self.observation_seq])
output_word = Word(
[Letter(symbol) for symbol in self.reward_trace])
self.OT.add_counterexample(input_word, output_word)
return True
return False
def findCounterExample(self):
"""Execute actions uniformly at random until we get a counter example"""
while True:
current_state_h = self.resetExploration()
for ep in range(STEPS_PER_EPISODE):
a = int(random.random() // (1 / (len(
self.world.map.availableActions(self.world.map.current)))))
(r_h, r_m, next_state_h, next_state_m,
obs) = self.executeOneStepExploration(current_state_h, a)
if obs != 'null':
self.observation_seq.append(obs)
self.reward_trace.append(r_m)
if self.isCounterExample(r_h, r_m):
self.nuof_counter_examples += 1
return None
else:
current_state_h = next_state_h
self.world.map.current = next_state_m
def isResetAction(self, a):
return int(a.__str__()) == len(self.world.map.actions)
def resetExploration(self):
self.observation_seq = []
self.reward_trace = []
self.world.map.reset()
self.world.mrm.reset()
return self.resetH()
def updateTraces(self, obs, r_m):
if obs != 'null':
self.observation_seq.append(obs)
self.reward_trace.append(r_m)
self.rewards += r_m
