def setUp(self):
self.pomdp = Tiger()
self.start = State(LEFT) # tiger behind left door
o = Observation()
h = History()
a = POMDPAction()
self.root = create_node(h, a, o)
self.root.inTree = True
params['c'] = 2
# setup a tree with root and its depth-1 children
self.root.create_children()
# expand listen-node
a = Action(listen=True)
s = self.start.clone()
o, r = a.do_on(s)
self.listen_child = self.root.children[a]
self.listen_child.h.add(a, o)
self.listen_child.inTree = True
self.listen_child.create_children()
# There is 3 depth-2 nodes, their histories go like:
# empty-listen-listen (ell), empty-listen-left (elf), empty-listen-right (elr)
# elr should have highest V (pref action)
for act, child in self.listen_child.children.items():
obs, r = child.a.do_on(s.clone())
self.listen_child.children[act] = create_node(child.h, act, obs)
self.listen_child.children[act].inTree = True
def __init__(self, max_iter, timeout, log=0, pref=True):
self.max_iter = max_iter
params['timeout'] = timeout
params['log'] = log
if not pref:
params["prefs"] = False
self.h = History()
self.last_action = POMDPAction()
self.first = True
def test_simulate_expansion(self):
root = create_node(History(), POMDPAction(), Observation())
#self.root1 = root
params.update({
'start_time': time.time(),
'gamma': 0.5,
'epsilon': 0.2,
'max_depth': 100,
'timeout': 3
})
simulate(self.start, root)
self.assertEqual(len(root.children), 3)
self.assertEqual(root.N, 1)
def run_policy(self, policy_param, num_episodes):
world = self.Env()
policy = self.Pol
policy.parameters = policy_param
J_hats = []
steps = []
states = []
actions = []
rewards = []
for n in range(num_episodes):
ret = 0
step = 0
states.append([])
actions.append([])
rewards.append([])
while not world.isEnd:
states[n].append(world.state)
step += 1
action = policy.sampleAction(world.state)
world.step(action)
discounted_reward = world.reward #* world.gamma**step
ret += discounted_reward
actions[n].append(action)
rewards[n].append(discounted_reward)
J_hats.append(ret)
self.results.append(ret)
steps.append(step)
world.reset()
D = []
for i in range(num_episodes):
history = History(states[i], actions[i], rewards[i])
D.append(history)
return np.mean(J_hats), D, np.max(J_hats)
def reset(self):
self.h = History()
self.last_action = POMDPAction()
self.first = True
np.random.seed(int(time.time()))
# Load data from CSV
state_dimension, num_actions, iOrder, theta_b, data_size, states, actions, rewards, pi_b = import_data(
"../data/test_data/data.csv")
print("Loaded data.")
print(state_dimension, num_actions, iOrder, theta_b, data_size)
D = []
# Parameter to select
c = 2
dOrder = 0
iteration_to_run = 1000
# Create Data set
for i in range(data_size):
history = History(states[i], actions[i], rewards[i])
D.append(history)
# Shuffle the data
np.random.shuffle(D)
D = D[:actual_data_size]
slog(len(D))
J_pi_b_hat = 0.0
for H in D:
J_pi_b_hat += np.sum(H.rewards)
J_pi_b_hat /= len(D)
slog(J_pi_b_hat)
policy = FBSoftmax(state_dimension, num_actions, iOrder, dOrder)
import os
params = {
'K': 50, # number of particles (size of the belief state space)
'c':
0, # exploration / exploitation ratio scalar constant (domain specific)
'epsilon': 0.0, # history discount factor
'gamma': 1, # reward discount factor
'R_lo': 0, # lowest value V(h) reached
'R_hi': 1, # highest value V(h) reached
'timeout': 120, # timeout for each iteration in seconds
'start_time': 0, # start time in seconds
'max_depth': 20, # max depth
'log': 1, # level of logs printed on console [0,2]
'prefs': True, # enable/disable prefered actions
'root': Node(POMDPAction(), History(), 0, 0, list())
}
# start time
start_time = 0
def UCB1_action_selection(node, greedy=False):
"""
Implementation of the UCB1 algorithm for solving a multi-armed bandit problem.
https://homes.di.unimi.it/~cesabian/Pubblicazioni/ml-02.pdf
Each action a available from the history h are assigned a value V(ha),
computed from simulations of the POMDP from the history h.
In non-greedy mode, this value is augmented by an exploration bonus for rarely-tried actions.
def setUp(self):
self.pomdp = Tiger()
self.start = State(LEFT) # tiger behind left door
self.o = Observation()
self.h = History()
self.a = POMDPAction()
def setUp(self):
self.b = Board(4, 5, 3)
self.s = State(self.b)
self.h = History()