def isCounterExample(self, r_h, r_m):
"""Return True if the two rewards r_h and r_m are different and add the counter example at the OT."""
if r_m != r_h:
print("CE", r_m, r_h, self.observation_seq)
input_word = Word(
[Letter(symbol) for symbol in self.observation_seq])
output_word = Word(
[Letter(symbol) for symbol in self.reward_trace])
self.OT.add_counterexample(input_word, output_word)
return True
return False
def deserialize(dict_data, possible_letters):
if dict_data is None:
raise Exception("dict_data cannot be None")
input_letter = Letter.deserialize(dict_data['input_letter'],
possible_letters)
output_letter = Letter.deserialize(dict_data['output_letter'],
possible_letters)
node = KnowledgeNode(input_letter, output_letter)
for child in dict_data["children"]:
child_node = KnowledgeNode.deserialize(child, possible_letters)
node.children[child_node.input_letter] = child_node
return node
def __init__(self, path, reset_cost, default_reward, value_expert):
self.world = Model(path, reset_cost, default_reward)
self.value_expert = value_expert
in_letters = [Letter(symbol) for symbol in self.world.mrm.observations]
self.kbase = MRMActiveKnowledgeBase(self.world)
self.OT = ObservationTable(input_letters=in_letters,
knowledge_base=self.kbase)
print('Initializing OT')
self.OT.initialize()
print('OT initialized')
#COUNTERS
self.total_learning_time = 0
self.total_exploring_time = 0
self.rewards = 0
self.iteration4Explor = 0
self.iteration4OT = 0
self.nuof_counter_examples = 0
#EXECUTION
self.learn()
while not self.check():
self.learn()
#END PRINT
self.endPrints()
remove(TMP_MODEL_PATH)
def submit_word(self, word):
"""This method return the Word produced by the target while submited the specified word"""
output_letters = []
for letter in word.letters:
symbols = letter.symbols
try:
output_symbols = []
for symbol in symbols:
try:
self.abstraction_layer.writeSymbol(symbol)
except ChannelDownException as e:
self._logger.debug("Channel is Down")
(curr_output_symbols,
data) = self.abstraction_layer.readSymbols()
output_symbols.extend(curr_output_symbols)
output_letters.append(Letter(symbols=output_symbols))
except Exception as e:
self._logger.fatal("An error occurred : {}".format(e))
output_letters.append(Letter(symbols=[EmptySymbol()]))
for i in range(len(word.letters)):
input_letter = word.letters[i]
output_letter = output_letters[i]
input_str = "None"
output_str = "None"
if input_letter.symbols is not None:
input_str = ','.join([s.name for s in input_letter.symbols])
if output_letter.symbols is not None:
output_str = ','.join([s.name for s in output_letter.symbols])
self._logger.debug(">>> {}".format(input_str))
self._logger.debug("<<< {}".format(output_str))
self.write_cache()
if self.submitted_word_cb is not None:
try:
self.submitted_word_cb(word.letters, output_letters)
except Exception as e:
self._logger.error(
"Error encountered while executed submitted_word_cb: {}".
format(e))
return Word(output_letters, normalize=False)
def _submit_letter(self, s, letter):
output_letter = EmptyLetter()
try:
to_send = ''.join([symbol for symbol in letter.symbols])
output_letter = Letter(self._send_and_receive(s, to_send))
except Exception as e:
self._logger.error(e)
return output_letter
def _execute_word(self, word):
# Executes the specified word.
if word is None:
raise Exception("Word cannot be None")
self._logger.debug("Execute word '{}'".format(word))
#print("Execute word '{}'".format(word))
reward_trace = []
self.nuof_MQs += 1
self.world.mrm.reset()
self.world.map.reset()
ge = GetExperience(self.world, word, MODE) #Create the scheduler
i = 0
while i < len(word):
letter = word.letters[i]
a = ge.getActionScheduler(self.world.map.current,
i) #ask for the next action to execute
(obs, rew) = self.world.moveAPF(a) #get observation and reward
reset = False
while Letter(obs) != letter:
if obs == "null": #if we observe nothing execute a new action
self.actionsForLearning += 1
a = ge.getActionScheduler(self.world.map.current, i)
(obs, rew) = self.world.moveAPF(a)
else: #if we observe something we don't want, reset
reset = True
break
if reset:
self.world.reset()
i = 0
reward_trace = []
else: #if we observe what we want
i += 1
reward_trace.append(rew)
w = Word([Letter(r) for r in reward_trace])
return w
def run(self):
self._logger.info("Configuring the inference process")
# creates letters for each input symbols
input_letters = [Letter(s) for s in self.input_symbols]
try:
# creates an abstraction layer on top of the channel to abstract and specialize received and sent messages
abstraction_layer = AbstractionLayer(channel=self.channel,
symbols=self.input_symbols +
self.output_symbols)
# creates a minimal adequat teacher
mat = GenericMAT(abstraction_layer=abstraction_layer,
process_wrapper=self.process_wrapper,
cache_file_path=os.path.join(
self.tmp_path, "cache.dump"))
# configures the RandomWalkMethod that will be used as an equivalence query
eqtests = RandomWalkMethod(knowledge_base=mat,
input_letters=input_letters,
max_steps=50000,
restart_probability=0.7)
# and finally, the LSTAR algorithm
self.lstar = LSTAR(input_vocabulary=self.input_symbols,
knowledge_base=mat,
max_states=30,
eqtests=eqtests)
# starts the inference process and stores the infered grammar in a dot file (graphviz)
self._logger.info("Starting the inference process...")
start_ts = time.time()
self.infered_automata = self.lstar.learn()
duration = time.time() - start_ts
self._logger.info(
"Inference process finished ({}s)".format(duration))
finally:
try:
self.process_wrapper.stop(force=True)
except Exception as e:
self._logger.info(
"Encountered the following error while stoping the process wrapper: {}"
.format(e))
def __init__(self, input_vocabulary, knowledge_base, max_states, tmp_dir=None, eqtests=None):
"""Implementation of the LSTAR algorithm.
Per default, WPMethod is used for equivalence tests. However, one can prefer a RandomWalkMethod
by specifying the following 'eqtests' parameter:
eqtests = RandomWalkMethod(self.knowledge_base, self.input_letters, 10000, 0.7)
"""
self.input_letters = [Letter(symbol) for symbol in input_vocabulary]
self.knowledge_base = knowledge_base
self.tmp_dir = tmp_dir
self.observation_table = ObservationTable(self.input_letters, self.knowledge_base)
self.max_states = max_states
self.eqtests = eqtests
self.__f_stop = False
def createNewMdp(self, state_id, current_obs):
"""Function which build the temporary MDP"""
to_add = []
state = self.inverseMappingState(state_id)
for action in self.model.map.availableActions(state):
action_id = self.mappingAction(action)
seen_obs = self.model.map.labelling[state_id][action_id]
for (next_state,
proba) in self.model.map.transitions[state_id][action_id]:
if seen_obs == "null": #we observe nothing
new_state = (next_state, current_obs)
if self.addNewTransition(new_state,
(state_id, current_obs),
action_id, proba):
to_add.append(new_state)
elif Letter(seen_obs) == self.observations.letters[
current_obs]: #we observe what we want
current_obs += 1
new_state = (next_state, current_obs)
if self.addNewTransition(new_state,
(state_id, current_obs - 1),
action_id, proba):
if len(self.observations
) == current_obs: #we have done
self.target.append(self.states.index(new_state))
else: #continue in every direction
to_add.append(new_state)
current_obs -= 1
else: #we observe something we don't want
self.transitions[self.states.index(
(state_id, current_obs))][action_id][0] = 1.0
self.reset_transitions[self.states.index(
(state_id, current_obs))].append(action_id)
return to_add
def findOptimalPlanToAnswer(self, observations, s):
plan = []
for obs in observations.letters:
maxEP = -1000000000
bestAct = None
bestState = dict()
for a in tmw.A:
maxProb = -1
expectedProb = 0
for ss in tmw.S:
if Letter(tmw.labelingFunc(ss)) == obs:
transProb = tmw.T(s, a, ss)
expectedProb += transProb
if transProb > maxProb:
maxProb = transProb
bestState[a] = ss
if expectedProb > maxEP:
maxEP = expectedProb
bestAct = a
plan.append(bestAct)
s = bestState[bestAct]
#return [Letter(symbol) for symbol in plan]
return plan
def __parse_graph_entry(graph_entry, states):
if graph_entry is None:
raise Exception("Graph entry cannot be None")
graph_entry = str(graph_entry).strip()
if len(graph_entry) == 0:
raise Exception("Graph entry cannot be None")
# parse first object
i_start_first_obj = graph_entry.find('"')
if i_start_first_obj == -1:
raise Exception("Cannot find first object definition")
i_end_first_obj = graph_entry.find('"', i_start_first_obj + 1)
if i_end_first_obj == -1:
raise Exception("Cannot find first object definition")
first_obj_name = graph_entry[i_start_first_obj +
1:i_end_first_obj].strip()
if len(first_obj_name) == 0:
raise Exception("Cannot parse the name of the first object")
first_state = None
for state in states:
if state.name == first_obj_name:
first_state = state
if first_state is None:
first_state = State(name=first_obj_name)
states.append(first_state)
remainder = graph_entry[i_end_first_obj + 1:].strip()
if remainder.startswith("->"):
remainder = remainder[2:].strip()
# parses the second state of the transition
i_start_second_obj = remainder.find('"')
if i_start_second_obj == -1:
raise Exception("Cannot find second object definition")
i_end_second_obj = remainder.find('"', i_start_second_obj + 1)
if i_end_second_obj == -1:
raise Exception("Cannot find second object definition")
second_obj_name = remainder[i_start_second_obj +
1:i_end_second_obj].strip()
if len(second_obj_name) == 0:
raise Exception(
"Cannot find the name of the destination state")
second_state = None
for state in states:
if state.name == second_obj_name:
second_state = state
if second_state is None:
second_state = State(name=second_obj_name)
states.append(second_state)
# parses the transition input and output letters
remainder = remainder[i_end_second_obj + 2:]
i_start_transition_details = remainder.find('[')
if i_start_transition_details == -1:
raise Exception("Cannot find transition details")
transition_details = remainder[i_start_transition_details:]
# parses the transition label
i_start_label = transition_details.find('label=')
if i_start_label == -1:
raise Exception("Cannot find label of the transition")
i_end_label = transition_details[i_start_label +
len('label="'):].find('"')
label = transition_details[i_start_label +
len('label="'):i_start_label +
len('label="') + i_end_label].strip()
if len(label) == 0:
raise Exception("Cannot find label")
# parses input and output letters out of the label
(input, output) = label.split('/')
input_letter = Letter(input.strip())
output_letter = Letter(output.strip())
# parses the transition name (url)
i_start_url = transition_details.find('URL=')
if i_start_url != -1:
i_end_url = transition_details[i_start_url +
len('URL="'):].find('"')
url = transition_details[i_start_url +
len('url="'):i_start_url +
len('url="') + i_end_url].strip()
t_name = url
else:
t_name = str(uuid.uuid4())
transition = Transition(t_name, second_state, input_letter,
output_letter)
first_state.transitions.append(transition)
def buildProductAutomaton(self, h):
"""Given a hypothesis of the angluin algo, build the product between the gird and this hypothesis and write it in a PRISM file.
The init state is {'c1','r1','null'} with no obs already made"""
rewards = "rewards\n"
labels = ''
out_file = open(TMP_MODEL_PATH, 'w')
#module
out_file.write("mdp\n\nmodule tmp\n\n")
#number of state and initial state
new_states = []
for s in self.world.map.states:
for o in range(len(h.get_states())):
labels += 'label "' + s + '_' + str(o) + '" = s=' + str(
len(new_states)) + ' ;\n'
new_states.append((s, o))
out_file.write("\ts : [0.." + str(len(new_states) - 1) + "] init " +
str(new_states.index((self.world.map.initiales[0],
0))) + ";\n\n")
#transitions
for s in new_states:
state_id = self.world.map.getIdState(s[0])
for a in self.world.map.availableActions(s[0]):
action_id = self.world.map.getIdAction(a)
obs = self.world.map.labelling[state_id][action_id]
#if len(self.world.map.transitions[state_id][action_id]) > 0:
out_file.write("\t[" + a + "] s=" + str(new_states.index(s)) +
"-> ")
temp_list = []
if obs == 'null':
rewards += "\t[" + a + "] (s=" + str(
new_states.index(s)) + ") : " + str(
self.world.mrm.default_reward) + ";\n"
for [dest, prob
] in self.world.map.transitions[state_id][action_id]:
index_dest = str(
new_states.index(
(self.world.map.getStateFromId(dest), s[1])))
temp_list.append(
str(prob) + " : (s'=" + index_dest + ")")
else:
tr_val = h.play_word(
Word([Letter(obs)]),
self.getStateInHypothesis(h.get_states(), s[1]))
state_in_h = int(tr_val[1][-1].name)
rewards += "\t[" + a + "] (s=" + str(
new_states.index(s)) + ") : " + str(
tr_val[0].last_letter().name) + ";\n"
for [dest, prob
] in self.world.map.transitions[state_id][action_id]:
index_dest = str(
new_states.index(
(self.world.map.getStateFromId(dest),
state_in_h)))
temp_list.append(
str(prob) + " : (s'=" + index_dest + ")")
out_file.write(" + ".join(temp_list))
out_file.write(";\n")
a = "reset"
out_file.write(
"\t[" + a + "] s=" + str(new_states.index(s)) +
"-> 1.0 : (s'=" +
str(new_states.index((self.world.map.initiales[0], 0))) +
");\n")
rewards += "\t[" + a + "] (s=" + str(
new_states.index(s)) + ") : " + str(
self.world.mrm.reset_cost) + ";\n"
out_file.write("\nendmodule\n\n")
out_file.write(labels)
rewards += "endrewards\n"
out_file.write(rewards)
out_file.close()