def __init__(self, path, reset_cost, default_reward, value_expert): self.world = Model(path, reset_cost, default_reward) self.value_expert = value_expert in_letters = [Letter(symbol) for symbol in self.world.mrm.observations] self.kbase = MRMActiveKnowledgeBase(self.world) self.OT = ObservationTable(input_letters=in_letters, knowledge_base=self.kbase) print('Initializing OT') self.OT.initialize() print('OT initialized') #COUNTERS self.total_learning_time = 0 self.total_exploring_time = 0 self.rewards = 0 self.iteration4Explor = 0 self.iteration4OT = 0 self.nuof_counter_examples = 0 #EXECUTION self.learn() while not self.check(): self.learn() #END PRINT self.endPrints() remove(TMP_MODEL_PATH)
def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None): """Implementation of the LSTAR algorithm. Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod by specifying the following 'eqtests' parameter: eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7) """ self.input_letters = [Letter(symbol) for symbol in input_vocabulary] self.knowledge_base = knowledge_base self.tmp_dir = tmp_dir self.observation_table = ObservationTable(self.input_letters, self.knowledge_base) self.max_states = max_states self.eqtests = eqtests self.__f_stop = False
def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None): """Implementation of the LSTAR algorithm. Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod by specifying the following 'eqtests' parameter: eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7) """ self.input_letters = [Letter(symbol) for symbol in input_vocabulary] self.knowledge_base = knowledge_base self.tmp_dir = tmp_dir self.observation_table = ObservationTable(self.input_letters, self.knowledge_base) self.max_states = max_states self.eqtests = eqtests
class LSTAR(object): """TODO : Describe here the inner working of the LSTAR Algorithm >>> from pylstar import LSTAR >>> from pylstar import State >>> from pylstar import Transition >>> from pylstar import Automata >>> from pylstar import Letter >>> from pylstar import FakeActiveKnowledgeBase >>> symbol_a = "a" >>> symbol_b = "b" >>> symbol_c = "c" >>> symbol_1 = 1 >>> symbol_2 = 2 >>> symbol_3 = 3 >>> l_a = Letter(symbol_a) >>> l_b = Letter(symbol_b) >>> l_c = Letter(symbol_c) >>> l_1 = Letter(symbol_1) >>> l_2 = Letter(symbol_2) >>> l_3 = Letter(symbol_3) >>> s0 = State("S0") >>> s1 = State("S1") >>> s2 = State("S2") >>> t1 = Transition("t1", output_state=s0, input_letter=l_a, output_letter=l_1) >>> t2 = Transition("t2", output_state=s1, input_letter=l_b, output_letter=l_2) >>> t3 = Transition("t3", output_state=s2, input_letter=l_c, output_letter=l_3) >>> s0.transitions = [t1, t2, t3] >>> t4 = Transition("t4", output_state=s1, input_letter=l_a, output_letter=l_2) >>> t5 = Transition("t5", output_state=s1, input_letter=l_b, output_letter=l_3) >>> t6 = Transition("t6", output_state=s0, input_letter=l_c, output_letter=l_1) >>> s1.transitions = [t4, t5, t6] >>> t7 = Transition("t7", output_state=s2, input_letter=l_a, output_letter=l_2) >>> t8 = Transition("t8", output_state=s2, input_letter=l_b, output_letter=l_3) >>> t9 = Transition("t9", output_state=s1, input_letter=l_c, output_letter=l_1) >>> s2.transitions = [t7, t8, t9] >>> automata = Automata(s0) >>> kbase = FakeActiveKnowledgeBase(automata) >>> input_vocabulary = [symbol_a, symbol_b, symbol_c] >>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5) >>> infered_automata = lstar.learn() >>> print(infered_automata.build_dot_code()) digraph "Automata" { "0" [shape=doubleoctagon, style=filled, fillcolor=white, URL="0"]; "2" [shape=ellipse, style=filled, fillcolor=white, URL="2"]; "1" [shape=ellipse, style=filled, fillcolor=white, URL="1"]; "0" -> "0" [fontsize=5, label="a / 1", URL="t0"]; "0" -> "1" [fontsize=5, label="b / 2", URL="t1"]; "0" -> "2" [fontsize=5, label="c / 3", URL="t2"]; "2" -> "2" [fontsize=5, label="a / 2", URL="t6"]; "2" -> "2" [fontsize=5, label="b / 3", URL="t7"]; "2" -> "1" [fontsize=5, label="c / 1", URL="t8"]; "1" -> "1" [fontsize=5, label="a / 2", URL="t3"]; "1" -> "1" [fontsize=5, label="b / 3", URL="t4"]; "1" -> "0" [fontsize=5, label="c / 1", URL="t5"]; } >>> from pylstar import LSTAR >>> from pylstar import State >>> from pylstar import Transition >>> from pylstar import Automata >>> from pylstar import Letter >>> from pylstar import FakeActiveKnowledgeBase >>> # input symbols >>> symbol_hello = "hello" >>> symbol_bye = "bye" >>> symbol_pass_valid = "pass valid" >>> symbol_pass_invalid = "pass invalid" >>> symbol_cmd1 = "cmd1" >>> symbol_cmd2 = "cmd2" >>> # output symbols >>> symbol_pass_request = "pass?" >>> symbol_ack = "ack" >>> symbol_welcome = "welcome" >>> symbol_error = "error" >>> # create a letter for each symbol >>> l_hello = Letter(symbol_hello) >>> l_bye = Letter(symbol_bye) >>> l_pass_valid = Letter(symbol_pass_valid) >>> l_pass_invalid = Letter("pass invalid") >>> l_cmd1 = Letter(symbol_cmd1) >>> l_cmd2 = Letter(symbol_cmd2) >>> l_welcome = Letter(symbol_welcome) >>> l_ack = Letter(symbol_ack) >>> l_pass_request = Letter(symbol_pass_request) >>> l_error = Letter(symbol_error) >>> # create the infered automata >>> s0 = State("S0") >>> s1 = State("S1") >>> s2 = State("S2") >>> t1 = Transition("t1", output_state=s1, input_letter=l_hello, output_letter=l_pass_request) >>> t2 = Transition("t2", output_state=s0, input_letter=l_bye, output_letter=l_ack) >>> t3 = Transition("t3", output_state=s0, input_letter=l_pass_valid, output_letter=l_error) >>> t4 = Transition("t4", output_state=s0, input_letter=l_pass_invalid, output_letter=l_error) >>> t5 = Transition("t5", output_state=s0, input_letter=l_cmd1, output_letter=l_error) >>> t6 = Transition("t6", output_state=s0, input_letter=l_cmd2, output_letter=l_error) >>> s0.transitions = [t1, t2, t3, t4, t5, t6] >>> t7 = Transition("t7", output_state=s1, input_letter=l_hello, output_letter=l_error) >>> t8 = Transition("t8", output_state=s0, input_letter=l_bye, output_letter=l_ack) >>> t9 = Transition("t9", output_state=s2, input_letter=l_pass_valid, output_letter=l_welcome) >>> t10 = Transition("t10", output_state=s1, input_letter=l_pass_invalid, output_letter=l_error) >>> t11 = Transition("t11", output_state=s1, input_letter=l_cmd1, output_letter=l_error) >>> t12 = Transition("t12", output_state=s1, input_letter=l_cmd2, output_letter=l_error) >>> s1.transitions = [t7, t8, t9, t10, t11, t12] >>> t13 = Transition("t13", output_state=s2, input_letter=l_hello, output_letter=l_error) >>> t14 = Transition("t14", output_state=s0, input_letter=l_bye, output_letter=l_ack) >>> t15 = Transition("t15", output_state=s2, input_letter=l_pass_valid, output_letter=l_error) >>> t16 = Transition("t16", output_state=s2, input_letter=l_pass_invalid, output_letter=l_error) >>> t17 = Transition("t17", output_state=s2, input_letter=l_cmd1, output_letter=l_ack) >>> t18 = Transition("t18", output_state=s2, input_letter=l_cmd2, output_letter=l_ack) >>> s2.transitions = [t13, t14, t15, t16, t17, t18] >>> automata = Automata(s0) >>> kbase = FakeActiveKnowledgeBase(automata) >>> input_vocabulary = [symbol_hello, symbol_bye, symbol_pass_valid, symbol_pass_invalid, symbol_cmd1, symbol_cmd2] >>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5) >>> infered_automata = lstar.learn() >>> print(infered_automata.build_dot_code()) digraph "Automata" { "0" [shape=doubleoctagon, style=filled, fillcolor=white, URL="0"]; "1" [shape=ellipse, style=filled, fillcolor=white, URL="1"]; "2" [shape=ellipse, style=filled, fillcolor=white, URL="2"]; "0" -> "1" [fontsize=5, label="hello / pass?", URL="t0"]; "0" -> "0" [fontsize=5, label="bye / ack", URL="t1"]; "0" -> "0" [fontsize=5, label="pass valid / error", URL="t2"]; "0" -> "0" [fontsize=5, label="pass invalid / error", URL="t3"]; "0" -> "0" [fontsize=5, label="cmd1 / error", URL="t4"]; "0" -> "0" [fontsize=5, label="cmd2 / error", URL="t5"]; "1" -> "1" [fontsize=5, label="hello / error", URL="t6"]; "1" -> "0" [fontsize=5, label="bye / ack", URL="t7"]; "1" -> "2" [fontsize=5, label="pass valid / welcome", URL="t8"]; "1" -> "1" [fontsize=5, label="pass invalid / error", URL="t9"]; "1" -> "1" [fontsize=5, label="cmd1 / error", URL="t10"]; "1" -> "1" [fontsize=5, label="cmd2 / error", URL="t11"]; "2" -> "2" [fontsize=5, label="hello / error", URL="t12"]; "2" -> "0" [fontsize=5, label="bye / ack", URL="t13"]; "2" -> "2" [fontsize=5, label="pass valid / error", URL="t14"]; "2" -> "2" [fontsize=5, label="pass invalid / error", URL="t15"]; "2" -> "2" [fontsize=5, label="cmd1 / ack", URL="t16"]; "2" -> "2" [fontsize=5, label="cmd2 / ack", URL="t17"]; } >>> from pylstar import LSTAR >>> from pylstar import State >>> from pylstar import Transition >>> from pylstar import Automata >>> from pylstar import Letter >>> from pylstar import FakeActiveKnowledgeBase >>> symbol_a = "a" >>> symbol_b = "b" >>> symbol_c = "c" >>> symbol_1 = 1 >>> symbol_2 = 2 >>> symbol_3 = 3 >>> l_a = Letter(symbol_a) >>> l_b = Letter(symbol_b) >>> l_c = Letter(symbol_c) >>> l_1 = Letter(symbol_1) >>> l_2 = Letter(symbol_2) >>> l_3 = Letter(symbol_3) >>> s0 = State("S0") >>> s1 = State("S1") >>> s2 = State("S2") >>> t1 = Transition("t1", output_state=s0, input_letter=l_a, output_letter=l_1) >>> t2 = Transition("t2", output_state=s1, input_letter=l_b, output_letter=l_2) >>> t3 = Transition("t3", output_state=s2, input_letter=l_c, output_letter=l_3) >>> s0.transitions = [t1, t2, t3] >>> t4 = Transition("t4", output_state=s2, input_letter=l_a, output_letter=l_3) >>> t5 = Transition("t5", output_state=s0, input_letter=l_b, output_letter=l_1) >>> t6 = Transition("t6", output_state=s1, input_letter=l_c, output_letter=l_2) >>> s1.transitions = [t4, t5, t6] >>> t7 = Transition("t7", output_state=s1, input_letter=l_a, output_letter=l_2) >>> t8 = Transition("t8", output_state=s2, input_letter=l_b, output_letter=l_3) >>> t9 = Transition("t9", output_state=s0, input_letter=l_c, output_letter=l_1) >>> s2.transitions = [t7, t8, t9] >>> automata = Automata(s0) >>> kbase = FakeActiveKnowledgeBase(automata) >>> input_vocabulary = [symbol_a, symbol_b, symbol_c] >>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5) >>> infered_automata = lstar.learn() >>> print(infered_automata.build_dot_code()) digraph "Automata" { "0" [shape=doubleoctagon, style=filled, fillcolor=white, URL="0"]; "2" [shape=ellipse, style=filled, fillcolor=white, URL="2"]; "1" [shape=ellipse, style=filled, fillcolor=white, URL="1"]; "0" -> "0" [fontsize=5, label="a / 1", URL="t0"]; "0" -> "1" [fontsize=5, label="b / 2", URL="t1"]; "0" -> "2" [fontsize=5, label="c / 3", URL="t2"]; "2" -> "1" [fontsize=5, label="a / 2", URL="t6"]; "2" -> "2" [fontsize=5, label="b / 3", URL="t7"]; "2" -> "0" [fontsize=5, label="c / 1", URL="t8"]; "1" -> "2" [fontsize=5, label="a / 3", URL="t3"]; "1" -> "0" [fontsize=5, label="b / 1", URL="t4"]; "1" -> "1" [fontsize=5, label="c / 2", URL="t5"]; } """ def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None): """Implementation of the LSTAR algorithm. Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod by specifying the following 'eqtests' parameter: eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7) """ self.input_letters = [Letter(symbol) for symbol in input_vocabulary] self.knowledge_base = knowledge_base self.tmp_dir = tmp_dir self.observation_table = ObservationTable(self.input_letters, self.knowledge_base) self.max_states = max_states self.eqtests = eqtests self.__f_stop = False def stop(self): """This method can be use to trigger the end of the learning process""" self._logger.info("Stopping the LSTAR learning process.") self.__f_stop = True def learn(self): self._logger.info("Starting the LSTAR learning process.") # intialization self.__initialize() f_hypothesis_is_valid = False i_round = 1 while not f_hypothesis_is_valid and not self.__f_stop: hypothesis = self.build_hypothesis(i_round) self.__serialize_hypothesis(i_round, hypothesis) counterexample = self.eqtests.find_counterexample(hypothesis) if counterexample is not None: self._logger.info("Counterexample '{}' found.".format(counterexample)) self.fix_hypothesis(counterexample) else: f_hypothesis_is_valid = True i_round += 1 self.__serialize_observation_table(i_round) self._logger.info("Automata successfully computed") return hypothesis def __serialize_hypothesis(self, i_round, hypothesis): if i_round is None: raise Exception("i_round cannot be None") if hypothesis is None: raise Exception("Hypothesis cannot be None") dot_code = hypothesis.build_dot_code() filepath = os.path.join(self.tmp_dir, "hypothesis_{}.dot".format(i_round)) with open(filepath, 'w') as fd: fd.write(dot_code) self._logger.info("Hypothesis produced on round '{}' stored in '{}'".format(i_round, filepath)) def __serialize_observation_table(self, i_round): if self.observation_table is None: raise Exception("Observation table cannot ne Bone") serialized_table = self.observation_table.serialize() str_date = datetime.strftime(datetime.now(), "%Y%m%d_%H%M%S") filepath = os.path.join(self.tmp_dir, "observation_table_{}_{}.raw".format(i_round, str_date)) with open(filepath, 'w') as fd: fd.write(serialized_table) self._logger.info("Observation table serialized in '{}'".format(filepath)) def fix_hypothesis(self, counterexample): if counterexample is None: raise Exception("counterexample cannot be None") self._logger.debug("fix hypothesis with counterexample '{}'".format(counterexample)) input_word = counterexample.input_word output_word = counterexample.output_word self.observation_table.add_counterexample(input_word, output_word) def build_hypothesis(self, i_round): if i_round is None: raise Exception("i_round cannot be None") f_consistent = False f_closed = False self._logger.info("Building the hypothesis ({} round)".format(i_round)) while not f_consistent or not f_closed: if not self.observation_table.is_closed(): self._logger.info("Observation table is not closed.") self.observation_table.close_table() f_closed = False else: self._logger.info("Observation table is closed") f_closed = True inconsistency = self.observation_table.find_inconsistency() if inconsistency is not None: self._logger.info("Observation table is not consistent.") self.observation_table.make_consistent(inconsistency) f_consistent = False else: self._logger.info("Observation table is consistent") f_consistent = True self.__serialize_observation_table(i_round) self._logger.info("Hypothesis computed") return self.observation_table.build_hypothesis() def __initialize(self): """Initialization of the observation table""" self.observation_table.initialize() self._logger.info("Observation table is initialized") self._logger.info("\n"+str(self.observation_table)) @property def input_vocabulary(self): """Input_vocabulary to use """ return self.__input_vocabulary @input_vocabulary.setter def input_vocabulary(self, input_vocabulary): if input_vocabulary is None: raise ValueError("Input_vocabulary cannot be None") if len(input_vocabulary) == 0: raise ValueError("Input vocabulary cannot be empty") self.__input_vocabulary = input_vocabulary @property def knowledge_base(self): """Membership Knowledge_base""" return self.__knowledge_base @knowledge_base.setter def knowledge_base(self, knowledge_base): if knowledge_base is None: raise ValueError("Knowledge_base cannot be None") self.__knowledge_base = knowledge_base @property def tmp_dir(self): """Temporary directory that host serialized observation tables and hypothesis""" return self.__tmp_dir @tmp_dir.setter def tmp_dir(self, value): if value is None: self.__tmp_dir = tempfile.mkdtemp(prefix='pylstar_') else: self.__tmp_dir = value @property def eqtests(self): return self.__eqtests @eqtests.setter def eqtests(self, eqtests): if eqtests is None: self.__eqtests = WpMethodEQ(self.knowledge_base, self.max_states, self.input_letters) else: self.__eqtests = eqtests
class LSTAR(object): """TODO : Describe here the inner working of the LSTAR Algorithm >>> from pylstar.LSTAR import LSTAR >>> from pylstar.automata.State import State >>> from pylstar.automata.Transition import Transition >>> from pylstar.automata.Automata import Automata >>> from pylstar.Letter import Letter >>> from pylstar.FakeActiveKnowledgeBase import FakeActiveKnowledgeBase >>> symbol_a = "a" >>> symbol_b = "b" >>> symbol_c = "c" >>> symbol_1 = 1 >>> symbol_2 = 2 >>> symbol_3 = 3 >>> l_a = Letter(symbol_a) >>> l_b = Letter(symbol_b) >>> l_c = Letter(symbol_c) >>> l_1 = Letter(symbol_1) >>> l_2 = Letter(symbol_2) >>> l_3 = Letter(symbol_3) >>> s0 = State("S0") >>> s1 = State("S1") >>> s2 = State("S2") >>> t1 = Transition("t1", output_state=s0, input_letter=l_a, output_letter=l_1) >>> t2 = Transition("t2", output_state=s1, input_letter=l_b, output_letter=l_2) >>> t3 = Transition("t3", output_state=s2, input_letter=l_c, output_letter=l_3) >>> s0.transitions = [t1, t2, t3] >>> t4 = Transition("t4", output_state=s1, input_letter=l_a, output_letter=l_2) >>> t5 = Transition("t5", output_state=s1, input_letter=l_b, output_letter=l_3) >>> t6 = Transition("t6", output_state=s0, input_letter=l_c, output_letter=l_1) >>> s1.transitions = [t4, t5, t6] >>> t7 = Transition("t7", output_state=s2, input_letter=l_a, output_letter=l_2) >>> t8 = Transition("t8", output_state=s2, input_letter=l_b, output_letter=l_3) >>> t9 = Transition("t9", output_state=s1, input_letter=l_c, output_letter=l_1) >>> s2.transitions = [t7, t8, t9] >>> automata = Automata(s0) >>> kbase = FakeActiveKnowledgeBase(automata) >>> input_vocabulary = [symbol_a, symbol_b, symbol_c] >>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5) >>> infered_automata = lstar.learn() >>> print infered_automata.build_dot_code() digraph G { "1,2,3,2" [shape=doubleoctagon, style=filled, fillcolor=white, URL="1,2,3,2"]; "2,3,1,2" [shape=ellipse, style=filled, fillcolor=white, URL="2,3,1,2"]; "2,3,1,1" [shape=ellipse, style=filled, fillcolor=white, URL="2,3,1,1"]; "1,2,3,2" -> "1,2,3,2" [fontsize=5, label="I='Letter('a')' / O='Letter(1)'", URL="t0"]; "1,2,3,2" -> "2,3,1,1" [fontsize=5, label="I='Letter('b')' / O='Letter(2)'", URL="t1"]; "1,2,3,2" -> "2,3,1,2" [fontsize=5, label="I='Letter('c')' / O='Letter(3)'", URL="t2"]; "2,3,1,2" -> "2,3,1,2" [fontsize=5, label="I='Letter('a')' / O='Letter(2)'", URL="t6"]; "2,3,1,2" -> "2,3,1,2" [fontsize=5, label="I='Letter('b')' / O='Letter(3)'", URL="t7"]; "2,3,1,2" -> "2,3,1,1" [fontsize=5, label="I='Letter('c')' / O='Letter(1)'", URL="t8"]; "2,3,1,1" -> "2,3,1,1" [fontsize=5, label="I='Letter('a')' / O='Letter(2)'", URL="t3"]; "2,3,1,1" -> "2,3,1,1" [fontsize=5, label="I='Letter('b')' / O='Letter(3)'", URL="t4"]; "2,3,1,1" -> "1,2,3,2" [fontsize=5, label="I='Letter('c')' / O='Letter(1)'", URL="t5"]; } >>> from pylstar.LSTAR import LSTAR >>> from pylstar.automata.State import State >>> from pylstar.automata.Transition import Transition >>> from pylstar.automata.Automata import Automata >>> from pylstar.Letter import Letter >>> from pylstar.FakeActiveKnowledgeBase import FakeActiveKnowledgeBase >>> # input symbols >>> symbol_hello = "hello" >>> symbol_bye = "bye" >>> symbol_pass_valid = "pass valid" >>> symbol_pass_invalid = "pass invalid" >>> symbol_cmd1 = "cmd1" >>> symbol_cmd2 = "cmd2" >>> # output symbols >>> symbol_pass_request = "pass?" >>> symbol_ack = "ack" >>> symbol_welcome = "welcome" >>> symbol_error = "error" >>> # create a letter for each symbol >>> l_hello = Letter(symbol_hello) >>> l_bye = Letter(symbol_bye) >>> l_pass_valid = Letter(symbol_pass_valid) >>> l_pass_invalid = Letter("pass invalid") >>> l_cmd1 = Letter(symbol_cmd1) >>> l_cmd2 = Letter(symbol_cmd2) >>> l_welcome = Letter(symbol_welcome) >>> l_ack = Letter(symbol_ack) >>> l_pass_request = Letter(symbol_pass_request) >>> l_error = Letter(symbol_error) >>> # create the infered automata >>> s0 = State("S0") >>> s1 = State("S1") >>> s2 = State("S2") >>> t1 = Transition("t1", output_state=s1, input_letter=l_hello, output_letter=l_pass_request) >>> t2 = Transition("t2", output_state=s0, input_letter=l_bye, output_letter=l_ack) >>> t3 = Transition("t3", output_state=s0, input_letter=l_pass_valid, output_letter=l_error) >>> t4 = Transition("t4", output_state=s0, input_letter=l_pass_invalid, output_letter=l_error) >>> t5 = Transition("t5", output_state=s0, input_letter=l_cmd1, output_letter=l_error) >>> t6 = Transition("t6", output_state=s0, input_letter=l_cmd2, output_letter=l_error) >>> s0.transitions = [t1, t2, t3, t4, t5, t6] >>> t7 = Transition("t7", output_state=s1, input_letter=l_hello, output_letter=l_error) >>> t8 = Transition("t8", output_state=s0, input_letter=l_bye, output_letter=l_ack) >>> t9 = Transition("t9", output_state=s2, input_letter=l_pass_valid, output_letter=l_welcome) >>> t10 = Transition("t10", output_state=s1, input_letter=l_pass_invalid, output_letter=l_error) >>> t11 = Transition("t11", output_state=s1, input_letter=l_cmd1, output_letter=l_error) >>> t12 = Transition("t12", output_state=s1, input_letter=l_cmd2, output_letter=l_error) >>> s1.transitions = [t7, t8, t9, t10, t11, t12] >>> t13 = Transition("t13", output_state=s2, input_letter=l_hello, output_letter=l_error) >>> t14 = Transition("t14", output_state=s0, input_letter=l_bye, output_letter=l_ack) >>> t15 = Transition("t15", output_state=s2, input_letter=l_pass_valid, output_letter=l_error) >>> t16 = Transition("t16", output_state=s2, input_letter=l_pass_invalid, output_letter=l_error) >>> t17 = Transition("t17", output_state=s2, input_letter=l_cmd1, output_letter=l_ack) >>> t18 = Transition("t18", output_state=s2, input_letter=l_cmd2, output_letter=l_ack) >>> s2.transitions = [t13, t14, t15, t16, t17, t18] >>> automata = Automata(s0) >>> kbase = FakeActiveKnowledgeBase(automata) >>> input_vocabulary = [symbol_hello, symbol_bye, symbol_pass_valid, symbol_pass_invalid, symbol_cmd1, symbol_cmd2] >>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5) >>> infered_automata = lstar.learn() >>> print infered_automata.build_dot_code() digraph G { "'pass?','ack','error','error','error','error'" [shape=doubleoctagon, style=filled, fillcolor=white, URL="'pass?','ack','error','error','error','error'"]; "'error','ack','welcome','error','error','error'" [shape=ellipse, style=filled, fillcolor=white, URL="'error','ack','welcome','error','error','error'"]; "'error','ack','error','error','ack','ack'" [shape=ellipse, style=filled, fillcolor=white, URL="'error','ack','error','error','ack','ack'"]; "'pass?','ack','error','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('hello')' / O='Letter('pass?')'", URL="t0"]; "'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('bye')' / O='Letter('ack')'", URL="t1"]; "'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('pass valid')' / O='Letter('error')'", URL="t2"]; "'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('pass invalid')' / O='Letter('error')'", URL="t3"]; "'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('cmd1')' / O='Letter('error')'", URL="t4"]; "'pass?','ack','error','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('cmd2')' / O='Letter('error')'", URL="t5"]; "'error','ack','welcome','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('hello')' / O='Letter('error')'", URL="t6"]; "'error','ack','welcome','error','error','error'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('bye')' / O='Letter('ack')'", URL="t7"]; "'error','ack','welcome','error','error','error'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('pass valid')' / O='Letter('welcome')'", URL="t8"]; "'error','ack','welcome','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('pass invalid')' / O='Letter('error')'", URL="t9"]; "'error','ack','welcome','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('cmd1')' / O='Letter('error')'", URL="t10"]; "'error','ack','welcome','error','error','error'" -> "'error','ack','welcome','error','error','error'" [fontsize=5, label="I='Letter('cmd2')' / O='Letter('error')'", URL="t11"]; "'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('hello')' / O='Letter('error')'", URL="t12"]; "'error','ack','error','error','ack','ack'" -> "'pass?','ack','error','error','error','error'" [fontsize=5, label="I='Letter('bye')' / O='Letter('ack')'", URL="t13"]; "'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('pass valid')' / O='Letter('error')'", URL="t14"]; "'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('pass invalid')' / O='Letter('error')'", URL="t15"]; "'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('cmd1')' / O='Letter('ack')'", URL="t16"]; "'error','ack','error','error','ack','ack'" -> "'error','ack','error','error','ack','ack'" [fontsize=5, label="I='Letter('cmd2')' / O='Letter('ack')'", URL="t17"]; } >>> from pylstar.LSTAR import LSTAR >>> from pylstar.automata.State import State >>> from pylstar.automata.Transition import Transition >>> from pylstar.automata.Automata import Automata >>> from pylstar.Letter import Letter >>> from pylstar.FakeActiveKnowledgeBase import FakeActiveKnowledgeBase >>> symbol_a = "a" >>> symbol_b = "b" >>> symbol_c = "c" >>> symbol_1 = 1 >>> symbol_2 = 2 >>> symbol_3 = 3 >>> l_a = Letter(symbol_a) >>> l_b = Letter(symbol_b) >>> l_c = Letter(symbol_c) >>> l_1 = Letter(symbol_1) >>> l_2 = Letter(symbol_2) >>> l_3 = Letter(symbol_3) >>> s0 = State("S0") >>> s1 = State("S1") >>> s2 = State("S2") >>> t1 = Transition("t1", output_state=s0, input_letter=l_a, output_letter=l_1) >>> t2 = Transition("t2", output_state=s1, input_letter=l_b, output_letter=l_2) >>> t3 = Transition("t3", output_state=s2, input_letter=l_c, output_letter=l_3) >>> s0.transitions = [t1, t2, t3] >>> t4 = Transition("t4", output_state=s2, input_letter=l_a, output_letter=l_3) >>> t5 = Transition("t5", output_state=s0, input_letter=l_b, output_letter=l_1) >>> t6 = Transition("t6", output_state=s1, input_letter=l_c, output_letter=l_2) >>> s1.transitions = [t4, t5, t6] >>> t7 = Transition("t7", output_state=s1, input_letter=l_a, output_letter=l_2) >>> t8 = Transition("t8", output_state=s2, input_letter=l_b, output_letter=l_3) >>> t9 = Transition("t9", output_state=s0, input_letter=l_c, output_letter=l_1) >>> s2.transitions = [t7, t8, t9] >>> automata = Automata(s0) >>> kbase = FakeActiveKnowledgeBase(automata) >>> input_vocabulary = [symbol_a, symbol_b, symbol_c] >>> lstar = LSTAR(input_vocabulary, kbase, max_states = 5) >>> infered_automata = lstar.learn() >>> print infered_automata.build_dot_code() digraph G { "1,2,3" [shape=doubleoctagon, style=filled, fillcolor=white, URL="1,2,3"]; "2,3,1" [shape=ellipse, style=filled, fillcolor=white, URL="2,3,1"]; "3,1,2" [shape=ellipse, style=filled, fillcolor=white, URL="3,1,2"]; "1,2,3" -> "1,2,3" [fontsize=5, label="I='Letter('a')' / O='Letter(1)'", URL="t6"]; "1,2,3" -> "3,1,2" [fontsize=5, label="I='Letter('b')' / O='Letter(2)'", URL="t7"]; "1,2,3" -> "2,3,1" [fontsize=5, label="I='Letter('c')' / O='Letter(3)'", URL="t8"]; "2,3,1" -> "3,1,2" [fontsize=5, label="I='Letter('a')' / O='Letter(2)'", URL="t3"]; "2,3,1" -> "2,3,1" [fontsize=5, label="I='Letter('b')' / O='Letter(3)'", URL="t4"]; "2,3,1" -> "1,2,3" [fontsize=5, label="I='Letter('c')' / O='Letter(1)'", URL="t5"]; "3,1,2" -> "2,3,1" [fontsize=5, label="I='Letter('a')' / O='Letter(3)'", URL="t0"]; "3,1,2" -> "1,2,3" [fontsize=5, label="I='Letter('b')' / O='Letter(1)'", URL="t1"]; "3,1,2" -> "3,1,2" [fontsize=5, label="I='Letter('c')' / O='Letter(2)'", URL="t2"]; } """ def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None): """Implementation of the LSTAR algorithm. Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod by specifying the following 'eqtests' parameter: eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7) """ self.input_letters = [Letter(symbol) for symbol in input_vocabulary] self.knowledge_base = knowledge_base self.tmp_dir = tmp_dir self.observation_table = ObservationTable(self.input_letters, self.knowledge_base) self.max_states = max_states self.eqtests = eqtests def learn(self): self._logger.info("Starting the LSTAR learning process.") # intialization self.__initialize() f_hypothesis_is_valid = False i_round = 1 while not f_hypothesis_is_valid: hypothesis = self.build_hypothesis(i_round) self.__serialize_hypothesis(i_round, hypothesis) counterexample = self.eqtests.find_counterexample(hypothesis) if counterexample is not None: self._logger.info("Counterexample '{}' found.".format(counterexample)) self.fix_hypothesis(counterexample) else: f_hypothesis_is_valid = True i_round += 1 self._logger.info("Automata successfully computed") return hypothesis def __serialize_hypothesis(self, i_round, hypothesis): if i_round is None: raise Exception("i_round cannot be None") if hypothesis is None: raise Exception("Hypothesis cannot be None") dot_code = hypothesis.build_dot_code() filepath = os.path.join(self.tmp_dir, "hypothesis_{}.dot".format(i_round)) with open(filepath, 'w') as fd: fd.write(dot_code) self._logger.info("Hypothesis produced on round '{}' stored in '{}'".format(i_round, filepath)) def __serialize_observation_table(self, i_round): if self.observation_table is None: raise Exception("Observation table cannot ne Bone") serialized_table = self.observation_table.serialize() str_date = datetime.strftime(datetime.now(), "%Y%m%d_%H%M%S") filepath = os.path.join(self.tmp_dir, "observation_table_{}_{}.raw".format(i_round, str_date)) with open(filepath, 'w') as fd: fd.write(serialized_table) self._logger.info("Observation table serialized in '{}'".format(filepath)) def fix_hypothesis(self, counterexample): if counterexample is None: raise Exception("counterexample cannot be None") self._logger.debug("fix hypothesis with counterexample '{}'".format(counterexample)) input_word = counterexample.input_word output_word = counterexample.output_word self.observation_table.add_counterexample(input_word, output_word) def build_hypothesis(self, i_round): if i_round is None: raise Exception("i_round cannot be None") f_consistent = False f_closed = False while not f_consistent or not f_closed: if not self.observation_table.is_closed(): self._logger.info("Observation table is not closed.") self.observation_table.close_table() f_closed = False else: self._logger.info("Observation table is closed") f_closed = True inconsistency = self.observation_table.find_inconsistency() if inconsistency is not None: self._logger.info("Observation table is not consistent.") self.observation_table.make_consistent(inconsistency) f_consistent = False else: self._logger.info("Observation table is consistent") f_consistent = True self.__serialize_observation_table(i_round) self._logger.info("Hypothesis computed") return self.observation_table.build_hypothesis() def __initialize(self): """Initialization of the observation table""" self.observation_table.initialize() self._logger.info("Observation table is initialized") self._logger.info("\n"+str(self.observation_table)) @property def input_vocabulary(self): """Input_vocabulary to use """ return self.__input_vocabulary @input_vocabulary.setter def input_vocabulary(self, input_vocabulary): if input_vocabulary is None: raise ValueError("Input_vocabulary cannot be None") if len(input_vocabulary) == 0: raise ValueError("Input vocabulary cannot be empty") self.__input_vocabulary = input_vocabulary @property def knowledge_base(self): """Membership Knowledge_base""" return self.__knowledge_base @knowledge_base.setter def knowledge_base(self, knowledge_base): if knowledge_base is None: raise ValueError("Knowledge_base cannot be None") self.__knowledge_base = knowledge_base @property def tmp_dir(self): """Temporary directory that host serialized observation tables and hypothesis""" return self.__tmp_dir @tmp_dir.setter def tmp_dir(self, value): if value is None: self.__tmp_dir = tempfile.mkdtemp(prefix='pylstar_') else: self.__tmp_dir = value @property def eqtests(self): return self.__eqtests @eqtests.setter def eqtests(self, eqtests): if eqtests is None: self.__eqtests = WpMethodEQ(self.knowledge_base, self.max_states, self.input_letters) else: self.__eqtests = eqtests
class LearningMRM: """Main class of the framework""" def __init__(self, path, reset_cost, default_reward, value_expert): self.world = Model(path, reset_cost, default_reward) self.value_expert = value_expert in_letters = [Letter(symbol) for symbol in self.world.mrm.observations] self.kbase = MRMActiveKnowledgeBase(self.world) self.OT = ObservationTable(input_letters=in_letters, knowledge_base=self.kbase) print('Initializing OT') self.OT.initialize() print('OT initialized') #COUNTERS self.total_learning_time = 0 self.total_exploring_time = 0 self.rewards = 0 self.iteration4Explor = 0 self.iteration4OT = 0 self.nuof_counter_examples = 0 #EXECUTION self.learn() while not self.check(): self.learn() #END PRINT self.endPrints() remove(TMP_MODEL_PATH) def learn(self): """Use L*_M algorithm to learn the MRM""" StartTime = time.time() iteration4OT = 0 closed = self.OT.is_closed() inconsistency = self.OT.find_inconsistency() while not closed or inconsistency is not None: iteration4OT += 1 print('Building the OT;', 'iteration', iteration4OT) if not closed: self.OT.close_table() if inconsistency is not None: self.OT.make_consistent(inconsistency) closed = self.OT.is_closed() inconsistency = self.OT.find_inconsistency() EndTime = time.time() self.total_learning_time += EndTime - StartTime def check(self): """Check if the hypothesis is correct""" StartTime = time.time() self.createHypothesis() if not self.passedFirstCheck(): # Expert value check res = False elif not self.passedSecondCheck(): # Exploitation check res = False else: res = True EndTime = time.time() self.total_exploring_time += EndTime - StartTime return res def endPrints(self): print() print('Optimization problem:', ["MIN", "MAX"][MODE]) print('# learning actions:', self.kbase.actionsForLearning) print() print('rewards:', self.rewards) print('nuof_MQs:', self.kbase.nuof_MQs) print('Exploration iterations:', self.iteration4Explor) print('nuof_counter_examples:', self.nuof_counter_examples) print('total_learning_time:', self.total_learning_time) print('total_exploring_time:', self.total_exploring_time) def createHypothesis(self): print() print('Building hypothesis MRM...') RM = self.OT.build_hypothesis() print('Hypothesis MRM built !') self.buildProductAutomaton( RM) # Write the prism file with the hypothesis program = stormpy.parse_prism_program(TMP_MODEL_PATH) properties = stormpy.parse_properties_for_prism_program( "Rmax=? [ LRA ]", program) options = stormpy.BuilderOptions(True, True) #To keep rewards and labels self.h = stormpy.build_sparse_model_with_options(program, options) self.result_h = stormpy.model_checking(self.h, properties[0], extract_scheduler=True).at(0) self.scheduler = stormpy.model_checking( self.h, properties[0], extract_scheduler=True).scheduler def passedFirstCheck(self): # Expert value check if self.result_h < self.value_expert: print(self.result_h, " < Value_expert:", self.value_expert, ", looking for counter-example...") self.findCounterExample() return False else: print(self.result_h, " >= Value_expert:", self.value_expert) return True def passedSecondCheck(self): # Exploitation check. Check if the hypothesis is correct by executing the strategy # After STERPS_PER_EPISODE we reset the system # We end if we observe a counter example or if we have already performed ACTIONTOTEXECUTE actions actionsExecuted = 0 while actionsExecuted < ACTIONTOEXECUTE: self.iteration4Explor += 1 print() print('Exploration iteration', self.iteration4Explor) # Initialize agent's state to a rand position and current_state_h = self.resetExploration() for step in range(STEPS_PER_EPISODE): a = self.scheduler.get_choice(current_state_h) if self.isResetAction(a): next_state_h = self.resetExploration() obs = "null" r_h = self.world.mrm.reset_cost r_m = self.world.mrm.reset_cost next_state_m = self.world.map.current else: (r_h, r_m, next_state_h, next_state_m, obs) = self.executeOneStepExploration( current_state_h, a.get_deterministic_choice()) self.updateTraces(obs, r_m) actionsExecuted += 1 if self.isCounterExample(r_h, r_m): self.nuof_counter_examples += 1 return False else: current_state_h = next_state_h self.world.map.current = next_state_m return True def getStateInHypothesis(self, states_h, state): for i in states_h: if int(i.name) == int(state): return i def buildProductAutomaton(self, h): """Given a hypothesis of the angluin algo, build the product between the gird and this hypothesis and write it in a PRISM file. The init state is {'c1','r1','null'} with no obs already made""" rewards = "rewards\n" labels = '' out_file = open(TMP_MODEL_PATH, 'w') #module out_file.write("mdp\n\nmodule tmp\n\n") #number of state and initial state new_states = [] for s in self.world.map.states: for o in range(len(h.get_states())): labels += 'label "' + s + '_' + str(o) + '" = s=' + str( len(new_states)) + ' ;\n' new_states.append((s, o)) out_file.write("\ts : [0.." + str(len(new_states) - 1) + "] init " + str(new_states.index((self.world.map.initiales[0], 0))) + ";\n\n") #transitions for s in new_states: state_id = self.world.map.getIdState(s[0]) for a in self.world.map.availableActions(s[0]): action_id = self.world.map.getIdAction(a) obs = self.world.map.labelling[state_id][action_id] #if len(self.world.map.transitions[state_id][action_id]) > 0: out_file.write("\t[" + a + "] s=" + str(new_states.index(s)) + "-> ") temp_list = [] if obs == 'null': rewards += "\t[" + a + "] (s=" + str( new_states.index(s)) + ") : " + str( self.world.mrm.default_reward) + ";\n" for [dest, prob ] in self.world.map.transitions[state_id][action_id]: index_dest = str( new_states.index( (self.world.map.getStateFromId(dest), s[1]))) temp_list.append( str(prob) + " : (s'=" + index_dest + ")") else: tr_val = h.play_word( Word([Letter(obs)]), self.getStateInHypothesis(h.get_states(), s[1])) state_in_h = int(tr_val[1][-1].name) rewards += "\t[" + a + "] (s=" + str( new_states.index(s)) + ") : " + str( tr_val[0].last_letter().name) + ";\n" for [dest, prob ] in self.world.map.transitions[state_id][action_id]: index_dest = str( new_states.index( (self.world.map.getStateFromId(dest), state_in_h))) temp_list.append( str(prob) + " : (s'=" + index_dest + ")") out_file.write(" + ".join(temp_list)) out_file.write(";\n") a = "reset" out_file.write( "\t[" + a + "] s=" + str(new_states.index(s)) + "-> 1.0 : (s'=" + str(new_states.index((self.world.map.initiales[0], 0))) + ");\n") rewards += "\t[" + a + "] (s=" + str( new_states.index(s)) + ") : " + str( self.world.mrm.reset_cost) + ";\n" out_file.write("\nendmodule\n\n") out_file.write(labels) rewards += "endrewards\n" out_file.write(rewards) out_file.close() def resetH(self): for s in range(len(self.h.states)): if {str(self.world.map.current) + '_0' }.issubset(self.h.states[s].labels): return s def getNextSateH(self, state, action): action = state.actions[action] r = random.random() c = 0 for transition in action.transitions: c += transition.value() if r < c: break return transition.column def getRewardH(self, state, action): c = 0 for i in range(state): # i: id in h => state in h => state in m c += len(self.h.states[i].actions) return self.h.reward_models[''].state_action_rewards[c + int( action.__str__())] # +1 because we have the reset action def fromIdStateHToIdStateM(self, sh): pattern = re.compile("s[0-9]+_[0-9]+") for i in self.h.states[sh].labels: if pattern.match(i): return i[:i.index('_')] def executeOneStepExploration(self, current_state_h, action): next_state_h = self.getNextSateH(self.h.states[current_state_h], action) r_h = self.getRewardH(current_state_h, action) obs = self.world.map.labelling[self.world.map.getIdState( self.world.map.current)][int(action.__str__())] next_state_m = self.fromIdStateHToIdStateM(next_state_h) #[next_state_m,obs] = self.world.map.moveFrom(self.world.map.current,self.world.map.actions[int(action.__str__())]) r_m = None if obs == 'null': r_m = self.world.mrm.default_reward else: r_m = self.world.mrm.move(obs) return (r_h, r_m, next_state_h, next_state_m, obs) def isCounterExample(self, r_h, r_m): """Return True if the two rewards r_h and r_m are different and add the counter example at the OT.""" if r_m != r_h: print("CE", r_m, r_h, self.observation_seq) input_word = Word( [Letter(symbol) for symbol in self.observation_seq]) output_word = Word( [Letter(symbol) for symbol in self.reward_trace]) self.OT.add_counterexample(input_word, output_word) return True return False def findCounterExample(self): """Execute actions uniformly at random until we get a counter example""" while True: current_state_h = self.resetExploration() for ep in range(STEPS_PER_EPISODE): a = int(random.random() // (1 / (len( self.world.map.availableActions(self.world.map.current))))) (r_h, r_m, next_state_h, next_state_m, obs) = self.executeOneStepExploration(current_state_h, a) if obs != 'null': self.observation_seq.append(obs) self.reward_trace.append(r_m) if self.isCounterExample(r_h, r_m): self.nuof_counter_examples += 1 return None else: current_state_h = next_state_h self.world.map.current = next_state_m def isResetAction(self, a): return int(a.__str__()) == len(self.world.map.actions) def resetExploration(self): self.observation_seq = [] self.reward_trace = [] self.world.map.reset() self.world.mrm.reset() return self.resetH() def updateTraces(self, obs, r_m): if obs != 'null': self.observation_seq.append(obs) self.reward_trace.append(r_m) self.rewards += r_m