next_q = next_qs.max(axis=1)
next_q.unchain()
target = self.gamma * next_q + reward
qs = self.qnet(state)
q = qs[:, action]
loss = F.mean_squared_error(target, q)
self.qnet.cleargrads()
loss.backward()
self.optimizer.update()
return loss.data
env = GridWorld()
agent = QLearningAgent()
episodes = 1000
loss_history = []
for episode in range(episodes):
state = env.reset()
state = one_hot(state)
total_loss, cnt = 0, 0
done = False
while not done:
action = agent.get_action(state)
next_state, reward, done = env.step(action)
next_state = one_hot(next_state)
if '__file__' in globals():
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import numpy as np
from common.gridworld import GridWorld
env = GridWorld()
V = {}
for state in env.states():
V[state] = np.random.randn()
env.render_v(V)
max_action = argmax(action_values)
action_probs = {0: 0, 1: 0, 2: 0, 3: 0}
action_probs[max_action] = 1.0
pi[state] = action_probs
return pi
def policy_iter(env, gamma, threshold=0.001, is_render=True):
pi = defaultdict(lambda: {0: 0.25, 1: 0.25, 2: 0.25, 3: 0.25})
V = defaultdict(lambda: 0)
while True:
V = policy_eval(pi, V, env, gamma, threshold)
new_pi = greedy_policy(V, env, gamma)
if is_render:
env.render_v(V, pi)
if new_pi == pi:
break
pi = new_pi
return pi
if __name__ == '__main__':
env = GridWorld()
gamma = 0.9
pi = policy_iter(env, gamma)
if done:
next_q = 0
rho = 1
else:
next_q = self.Q[next_state, next_action]
rho = self.pi[next_state][next_action] / self.b[next_state][
next_action]
target = rho * (reward + self.gamma * next_q)
self.Q[state, action] += (target - self.Q[state, action]) * self.alpha
self.pi[state] = greedy_probs(self.Q, state, 0)
self.b[state] = greedy_probs(self.Q, state, self.epsilon)
env = GridWorld()
agent = SarsaOffPolicyAgent()
episodes = 10000
for episode in range(episodes):
state = env.reset()
agent.reset()
while True:
action = agent.get_action(state)
next_state, reward, done = env.step(action)
agent.update(state, action, reward, done)
if done:
agent.update(next_state, None, None, None)
V[state] = new_V
return V
def policy_eval(pi, V, env, gamma, threshold=0.001):
while True:
old_V = V.copy()
V = eval_onestep(pi, V, env, gamma)
delta = 0
for state in V.keys():
t = abs(V[state] - old_V[state])
if delta < t:
delta = t
if delta < threshold:
break
return V
if __name__ == '__main__':
env = GridWorld()
gamma = 0.9
pi = defaultdict(lambda: {0: 0.25, 1: 0.25, 2: 0.25, 3: 0.25})
V = defaultdict(lambda: 0)
V = policy_eval(pi, V, env, gamma)
env.render_v(V, pi)