class AdaptiveBetaQLearning(BoltzmannQLearning):
def __init__(self, buffer_size=5, sigma=1., delta=1, inverse_temp=True):
"""
:param buffer: buffer size for historical TDE
:param sigma: sigma parameter for beta function
:param delta: delta parater for beta update, default 1: fastest adaptation of exploration
:param inverse_temp: whether optimizes for beta (inverse temperature) or for sigmoid (temperature
parameter)
"""
super().__init__(self)
self.TDE = FIFOQueue(capacity=buffer_size)
self.sigma = sigma
if inverse_temp:
self.__to_sigmoid = lambda x: 1 / x
self.update_exploration = self.__update_beta
self.explore = EPSILON
else:
self.__to_sigmoid = lambda x: EPSILON if x == 0 else x
self.update_exploration = self.__update_temp
self.explore = 1
self.delta = delta
def __update_temp(self):
self.explore += self.delta * (np.exp(self.sigma * self.TDE.abs_avg()) -
1 - self.explore)
def __update_beta(self):
self.explore += self.delta * (
1 / (np.exp(self.sigma * self.TDE.abs_avg()) - 1 + EPSILON) -
self.explore)
def __calculate_sigmoid(self):
"""
Function of temperature in BoltzmannQLearning. Modified here to
SOFTMAX VBDE: f(s, \sigma) = \frac{e^{-\sigma |TDE|}}{1-e^{-\sigma |TDE|}}
:return:
"""
return self.__to_sigmoid(self.explore)
def update_q(self, state_key, action_key, reward_value, next_max_q):
'''
Update Q-Value while at the mean time update
Args:
state_key: The key of state.
action_key: The key of action.
reward_value: R-Value(Reward).
next_max_q: Maximum Q-Value.
'''
# Now Q-Value.
q = self.extract_q_df(state_key, action_key)
self.TDE.add(reward_value + self.gamma_value * next_max_q - q)
self.update_exploration()
super(BoltzmannQLearning).update_q(state_key, action_key, reward_value,
next_max_q)
def __init__(self, instructions, input_=None):
super().__init__(instructions)
self.opcodes = {
1: self.add,
2: self.multiply,
99: self.halt,
3: self.input,
4: self.output,
5: self.jump_if_true,
6: self.jump_if_false,
7: self.less_than,
8: self.equals,
}
self.input_data = FIFOQueue(input_)
self.output_data = FIFOQueue()
def breadth_first_tree_search(problem):
"""Експандирај го прво најплиткиот јазол во пребарувачкото дрво.
:param problem: даден проблем
:return: Node
"""
return tree_search(problem, FIFOQueue())
def breadth_first_graph_search(problem):
"""Експандирај го прво најплиткиот јазол во пребарувачкиот граф.
:param problem: даден проблем
:return: Node
"""
return graph_search(problem, FIFOQueue())
def breadth_first_search(problem, observation):
"""[Figure 3.11]"""
node = Node(problem.initial)
if len(observation) == 1:
return node
frontier = FIFOQueue()
frontier.append(node)
explored = set()
all_games = []
while frontier:
node = frontier.pop()
explored.add(node.state)
new_check = []
check = node.expand(problem, observation)
for i in range(len(check)):
if check[i] is not None:
new_check.append(check[i])
if len(new_check) > 0:
for child in new_check:
if child.state not in explored and child not in frontier:
if len(
observation
) - 1 == child.depth: # במקום זה בודקים אם הגענו לעומק האחרון
current_child = []
temp_ch = child
for ch in range(len(observation)
): # if there is a problem check here
current_child.append(temp_ch)
temp_ch = child.parent
all_games.append(current_child)
frontier.append(child)
return all_games
def breadth_first_search(problem):
"""[Figure 3.11]"""
node = Node(problem.initial)
initial_time = time()
visitados = 0
if problem.goal_test(node.state):
print(node)
return node
frontier = FIFOQueue()
frontier.append(node)
explored = set()
while frontier:
visitados += len(frontier)
node = frontier.pop()
explored.add(node.state)
for child in node.expand(problem):
if child.state not in explored and child not in frontier:
visitados += 1
if problem.goal_test(child.state):
for i in child.path():
timespent = time()
return [
child,
child.path(), visitados,
len(explored), timespent - initial_time,
child.path_cost
]
frontier.append(child)
return None
def breadth_first_search(problem):
"""[Figure 3.11]"""
global frontier, node, explored, counter
if counter == -1:
node = Node(problem.initial)
display_current(node)
if problem.goal_test(node.state):
return node
frontier = FIFOQueue()
frontier.append(node)
display_frontier(frontier)
explored = set()
if counter % 3 == 0 and counter >= 0:
node = frontier.pop()
display_current(node)
explored.add(node.state)
if counter % 3 == 1 and counter >= 0:
for child in node.expand(problem):
if child.state not in explored and child not in frontier:
if problem.goal_test(child.state):
return child
frontier.append(child)
display_frontier(frontier)
if counter % 3 == 2 and counter >= 0:
display_explored(node)
return None
def hierarchical_search(problem, hierarchy):
"""
[Figure 11.5] 'Hierarchical Search, a Breadth First Search implementation of Hierarchical
Forward Planning Search'
The problem is a real-world prodlem defined by the problem class, and the hierarchy is
a dictionary of HLA - refinements (see refinements generator for details)
"""
act = Node(problem.actions[0])
frontier = FIFOQueue()
frontier.append(act)
while(True):
if not frontier:
return None
plan = frontier.pop()
print(plan.state.name)
hla = plan.state # first_or_null(plan)
prefix = None
if plan.parent:
prefix = plan.parent.state.action # prefix, suffix = subseq(plan.state, hla)
outcome = Problem.result(problem, prefix)
if hla is None:
if outcome.goal_test():
return plan.path()
else:
print("else")
for sequence in Problem.refinements(hla, outcome, hierarchy):
print("...")
frontier.append(Node(plan.state, plan.parent, sequence))
def breadth_first_search(problem):
"""[Figure 3.11]"""
node = Node(problem.initial)
# if problem.goal_test(node.state):
# return node
frontier = FIFOQueue()
frontier.append(node)
explored = set()
while frontier:
node = frontier.pop()
# print("here")
explored.add(node.state)
# print("explored:", explored)
children_list = node.expand(problem)
# print("children list:", children_list, "of node:", node.state)
valid_kids_list = create_valid_kids_list(children_list, problem)
# print("valid kids:", valid_kids_list)
final_children = []
for child in valid_kids_list:
if child.state not in explored and child not in frontier:
if child.depth == len(problem.observations) - 1:
final_children.append(child)
# return child
else:
frontier.append(child)
return final_children
def waterfall_levels(csp, var, val, assignment, removals, inferlist):
"""
Apply a waterfall of inferences
Add whatever inference you like into all_inferences
The waterfall will stop until no more effects incurr
"""
all_inferences = inferlist
inferences = FIFOQueue(items=all_inferences)
while inferences:
inference = inferences.pop()
tmp = removals.copy()
if not inference(csp, var, val, assignment, removals):
return False
if tmp != removals:
for next_inference in all_inferences:
if next_inference != inference:
inferences.append(next_inference)
return True
def breadth_first_search(problem):
"""[Figure 3.11]"""
node_count = 0
leaf_counter = 0
game = MedicalProblem(problem)
Pteam, Mteam = game.police, game.medical
if len(game.map_list) == 1:
return answer_queries_init(game)
grey = FIFOQueue()
leafs_queries = []
coordination, permutation = game.combinations(game.map_list[0])
for i in permutation:
root_state = fill_q_mark(game.map_list[0], coordination, i)
grey.append(Node(root_state))
node_count += 1
while grey:
node_to_explore = grey.pop()
if len(game.map_list) - 1 > node_to_explore.depth:
actions = node_to_explore.actions(node_to_explore.state, Pteam,
Mteam)
for action in actions:
cur_map = node_to_explore.result(node_to_explore.state, action,
Pteam, Mteam)
turn = node_to_explore.depth
child_template = game.map_list[turn + 1]
if can_be_identical(child_template, cur_map):
new_node = Node(cur_map, node_to_explore, action)
node_count += 1
if new_node not in grey:
grey.append(new_node)
else:
leaf_counter += 1
leafs_queries.append(answer_queries(node_to_explore, game.queries))
if len(leafs_queries) == 0:
return answer_queries_false(game.queries)
else:
sol = {}
for j in range(len(leafs_queries[0])):
Found_True = False
Found_False = False
for i in range(len(leafs_queries)):
if leafs_queries[i][j]:
Found_True = True
else:
Found_False = True
if Found_True and Found_False:
sol[game.queries[j]] = '?'
elif Found_True:
sol[game.queries[j]] = 'T'
elif Found_False:
sol[game.queries[j]] = 'F'
else:
sol[game.queries[j]] = 'error'
break
return sol
def __init__(self, buffer_size=5, sigma=1., delta=1, inverse_temp=True):
"""
:param buffer: buffer size for historical TDE
:param sigma: sigma parameter for beta function
:param delta: delta parater for beta update, default 1: fastest adaptation of exploration
:param inverse_temp: whether optimizes for beta (inverse temperature) or for sigmoid (temperature
parameter)
"""
super().__init__(self)
self.TDE = FIFOQueue(capacity=buffer_size)
self.sigma = sigma
if inverse_temp:
self.__to_sigmoid = lambda x: 1 / x
self.update_exploration = self.__update_beta
self.explore = EPSILON
else:
self.__to_sigmoid = lambda x: EPSILON if x == 0 else x
self.update_exploration = self.__update_temp
self.explore = 1
self.delta = delta
def breadth_first_search(problem):
"""[Figure 3.11]"""
global frontier, node, explored, counter
if counter == -1:
node = Node(problem.initial)
display_current(node)
if problem.goal_test(node.state):
return node
frontier = FIFOQueue()
frontier.append(node)
display_frontier(frontier)
explored = set()
if counter % 3 == 0 and counter >= 0:
node = frontier.pop()
display_current(node)
explored.add(node.state)
if counter % 3 == 1 and counter >= 0:
for child in node.expand(problem):
if child.state not in explored and child not in frontier:
if problem.goal_test(child.state):
return child
frontier.append(child)
display_frontier(frontier)
if counter % 3 == 2 and counter >= 0:
display_explored(node)
return None
def hierarchical_search(problem, hierarchy):
"""
[Figure 11.5] 'Hierarchical Search, a Breadth First Search implementation of Hierarchical
Forward Planning Search'
The problem is a real-world prodlem defined by the problem class, and the hierarchy is
a dictionary of HLA - refinements (see refinements generator for details)
"""
act = Node(problem.actions[0])
frontier = FIFOQueue()
frontier.append(act)
while(True):
if not frontier: #(len(frontier)==0):
return None
plan = frontier.pop()
print(plan.state.name)
hla = plan.state #first_or_null(plan)
prefix = None
if plan.parent:
prefix = plan.parent.state.action #prefix, suffix = subseq(plan.state, hla)
outcome = Problem.result(problem, prefix)
if hla is None:
if outcome.goal_test():
return plan.path()
else:
print("else")
for sequence in Problem.refinements(hla, outcome, hierarchy):
print("...")
frontier.append(Node(plan.state, plan.parent, sequence))
def breadth_first_search(problem, num_observe, board, dict_bfs):
node = Node(problem.initial)
frontier = FIFOQueue()
frontier.append(node)
explored = set()
tmp_board = tuple([tuple(x) for x in board])
while frontier:
node = frontier.pop()
explored.add(node.state)
for child in node.expand(problem, num_observe, problem.obs_tup):
if child.state not in explored and child not in frontier:
if child.depth == num_observe - 1:
path = child.find_dynasty()
dict_bfs[tmp_board].append(path)
frontier.append(child)
return None
def breadth_first_search(problem, max_depth, results_dict, init_state,
observations):
node = Node(problem.initial)
frontier = FIFOQueue()
frontier.append(node)
explored = set()
while frontier:
node = frontier.pop()
explored.add(node.state)
for child in node.expand(problem, observations):
if child.state not in explored and child not in frontier:
if child.depth == max_depth - 1:
path = child.path()
results_dict[init_state].append(path)
frontier.append(child)
return None
def breadth_first_search(problem):
node = Node(problem.initial)
if problem.goal_test(node.state):
return node
frontier = FIFOQueue()
frontier.append(node)
explored = set()
while frontier:
node = frontier.pop()
explored.add(node.state)
for child in node.expand(problem):
if child.state not in explored and child not in frontier:
if problem.goal_test(child.state):
return child
frontier.append(child)
return None
def breadth_first_search(problem):
"""[Figure 3.11]"""
node = Node(problem.initial)
frontier = FIFOQueue()
frontier.append(node)
explored = set()
child_path = []
while frontier:
node = frontier.pop()
explored.add(node.state)
for child in node.expand(problem):
if child.state not in explored and child not in frontier:
if child.depth == len(problem.observation_list) - 1:
child_path.append(child.path())
frontier.append(child)
return child_path
def breadth_first_search(problem):
"""[Figure 3.11]"""
node = Node(problem.initial)
total_nodes = 0
if problem.goal_test(node.state):
return node, total_nodes
frontier = FIFOQueue()
frontier.append(node)
explored = set()
while frontier:
node = frontier.pop()
total_nodes += 1
explored.add(node.state)
for child in node.expand(problem):
if child.state not in explored and child not in frontier:
if problem.goal_test(child.state):
return child, total_nodes
frontier.append(child)
return None, total_nodes
def breadth_first_search(problem):
"[Fig. 3.11]"
node = Node(problem.initial)
if problem.goal_test(node.state):
return node
frontier = FIFOQueue()
frontier.append(node)
explored = set()
while frontier:
node = frontier.pop()
explored.add(node.state)
for child in node.expand(problem):
if child.state not in explored and child not in frontier:
if problem.goal_test(child.state):
return child
frontier.append(child)
return None
def breadth_first_search(problem):
"[Figure 3.11]"
node = Node(problem.initial)
if problem.goal_test(node.state):
return node
frontier = FIFOQueue()
frontier.append(node)
explored = set()
while frontier:
node = frontier.pop()
try:
explored.add(node.state)
except:
traceback.print_exc()
sys.exit(1)
for child in node.expand(problem):
if child.state not in explored and child not in frontier:
if problem.goal_test(child.state):
return child
frontier.append(child)
return None
def breadth_first_tree_search(problem):
"""Search the shallowest nodes in the search tree first."""
global frontier, counter
if counter == -1:
frontier = FIFOQueue()
return tree_search(problem)
def breadth_first_tree_search(problem):
"""Search the shallowest nodes in the search tree first."""
return tree_search(problem, FIFOQueue())
def breadth_first_search(problem):
"Search the shallowest nodes in the search tree first."
return graph_search(problem, FIFOQueue())
class IntCodeComputer2(IntCodeComputer):
def __init__(self, instructions, input_=None):
super().__init__(instructions)
self.opcodes = {
1: self.add,
2: self.multiply,
99: self.halt,
3: self.input,
4: self.output,
5: self.jump_if_true,
6: self.jump_if_false,
7: self.less_than,
8: self.equals,
}
self.input_data = FIFOQueue(input_)
self.output_data = FIFOQueue()
def get_opcode(self):
value = self.instructions[self.cursor]
str_ = '{:05d}'.format(value)
modes = str_[:-2]
code = int(str_[-2:])
modes = tuple(reversed([int(c) for c in modes]))
return self.opcodes[code], modes
# noinspection PyMethodOverriding
def get(self, index, mode):
if mode == 0:
return self.instructions[self.instructions[index]]
elif mode == 1:
return self.instructions[index]
else:
raise ValueError
# noinspection PyMethodOverriding
def set(self, index, value, mode):
if mode == 0:
self.instructions[self.instructions[index]] = value
elif mode == 1:
self.instructions[index] = value
else:
raise ValueError
@opcode(1)
def input(self, modes):
self.set(self.cursor + 1, self.input_data.pop(), modes[0])
@opcode(1)
def output(self, modes):
self.output_data.push(self.get(self.cursor + 1, modes[0]))
@opcode(3)
def add(self, modes):
self.set(
self.cursor + 3,
self.get(self.cursor + 1, modes[0]) +
self.get(self.cursor + 2, modes[1]), modes[2])
@opcode(3)
def multiply(self, modes):
self.set(
self.cursor + 3,
self.get(self.cursor + 1, modes[0]) *
self.get(self.cursor + 2, modes[1]), modes[2])
@opcode(2)
def jump_if_true(self, modes):
if self.get(self.cursor + 1, modes[0]) != 0:
self.cursor = self.get(self.cursor + 2, modes[1])
self.cursor -= 3 # to undo the increment that happens automatically due to the opcode decorator
@opcode(2)
def jump_if_false(self, modes):
if self.get(self.cursor + 1, modes[0]) == 0:
self.cursor = self.get(self.cursor + 2, modes[1])
self.cursor -= 3 # to undo the increment that happens automatically due to the opcode decorator
@opcode(3)
def less_than(self, modes):
self.set(
self.cursor + 3,
int(
self.get(self.cursor +
1, modes[0]) < self.get(self.cursor + 2, modes[1])),
modes[2])
@opcode(3)
def equals(self, modes):
self.set(
self.cursor + 3,
int(
self.get(self.cursor +
1, modes[0]) == self.get(self.cursor + 2, modes[1])),
modes[2])
node = l.pop()
# print(node)
if problem.goal_test(node.state):
return node
explored.add(node.state)
frontier.extend(child for child in node.expand(problem)
if child.state not in explored and
child not in frontier)
return None
def breadth_first_`````````````````````````````````````````````````````````````(problem):
"Search the shallowest nodes in the search tree first."
return tree_search(problem, FIFOQueue())
def depth_first_tree_search(problem):
"Search the deepest nodes in the search tree first."
return tree_search(problem, Stack())
def depth_first_graph_search(problem):
"Search the deepest nodes in the search tree first."
return graph_search(problem, Stack())
def breadth_first_search(problem):
"[Figure 3.11]"
node = Node(problem.initial)
def compute_states(self):
# BFS
a_eval = deepcopy(self.eval)
frontier = FIFOQueue()
min_front_reward = 0
frontier.append(a_eval) # Eval
explored = set() # ABSTRACT ( Eval.state ) + MD
possible_state = set() # we return this so we don't have any keyerror.
possible_state.add(("DEADEND"))
for i in possible_state:
print(i)
T = {}
R = {}
start_time = time.time()
flag = True
while flag and frontier and time.time() - start_time < 40:
temp_eval = frontier.pop()
explored.add(
self.eval_state_to_ab_state_plus_md(temp_eval)) # add state.
# explore possible children of the board if they exist.
if "pacman" in temp_eval.special_things and temp_eval.special_things[
"pacman"] is not 'dead':
before_action_h_value = self.set_h(temp_eval)
#print(temp_eval.special_things)
#print(before_action_h_value)
parent_state_md = self.eval_state_to_ab_state_plus_md(
temp_eval)
parent_state_reward = temp_eval.accumulated_reward
# children
for action in ["U", "L", "R", "D"]:
#print(action)
#parent
#child. Copy, move, get state, get reward.
child_eval = deepcopy(temp_eval)
checker.Evaluator.change_state_after_action(
child_eval, action)
next_state_md = self.eval_state_to_ab_state_plus_md(
child_eval)
after_action_reward = child_eval.accumulated_reward
# Did the action finish the board?
if after_action_reward - parent_state_reward >= 30:
flag = False # found a solution
print('SOMETHING STRANGE')
empty_state = deepcopy(temp_eval)
if action == "R":
#print("RIGHT")
a = 0
b = 1
elif action == "L":
#print("LEFT")
a = 0
b = -1
elif action == "U":
print("UP")
a = -1
b = 0
elif action == "D":
#print("DOWN")
a = 1
b = 0
else:
# ERROR
#print("There is an error")
a = 0
b = 0
# update pacman's location
old_pacman_location = temp_eval.special_things[
"pacman"]
new_pacman_location = (
temp_eval.special_things["pacman"][0] + a,
temp_eval.special_things["pacman"][1] + b)
empty_state.special_things[
"pacman"] = new_pacman_location
empty_state.state[old_pacman_location] = 10
empty_state.state[new_pacman_location] = 66
R[self.eval_state_to_ab_state_plus_md(
empty_state)] = after_action_reward * 100
print("REWARD %s " % after_action_reward)
T[(parent_state_md, action)] = (
1,
self.eval_state_to_ab_state_plus_md(empty_state))
explored.add(
self.eval_state_to_ab_state_plus_md(empty_state))
print(self.eval_state_to_ab_state_plus_md(empty_state))
print("ADDED")
print(len(possible_state))
possible_state.add(
self.eval_state_to_ab_state_plus_md(empty_state))
print(len(possible_state))
elif child_eval.special_things["pacman"] is not 'dead':
R[next_state_md] = self.set_h(temp_eval)
T[(parent_state_md, action)] = (1, next_state_md)
possible_state.add(next_state_md)
#explored.add(next_state_md)
# we dont want to add states where pacman is dead:
if "pacman" in child_eval.special_things and child_eval.special_things[
"pacman"] is not 'dead':
if next_state_md not in explored and child_eval not in frontier:
if child_eval.accumulated_reward >= min_front_reward - 1:
min_front_reward = max(
child_eval.accumulated_reward,
min_front_reward)
frontier.append(child_eval)
print("Min Reward %s" % min_front_reward)
print("Finished loop: States: " + str(len(explored)) + " Queue: " +
str(len(frontier)))
print("SIZE OF EXPLORED = %s SIZE OF T = %s SIZE OF R = %s" %
(len(explored), len(T), len(R)))
return possible_state, T, R
def breadth_first_graph_search(problem):
return graph_search(problem, FIFOQueue())
def breadth_first_tree_search(problem):
return tree_search(problem, FIFOQueue())