def sarsa(self, YuanYangEnv, num_iter, alpha, epsilon):
for iter in range(num_iter):
#随机初始化状态
s = YuanYangEnv.reset()
#随机选初始动作
a = YuanYangEnv.actions[int(random.random() *
len(YuanYangEnv.actions))]
t = False
count = 0
while False == t and count < 200:
#与环境交互得到下一个状态
s_next, r, t = YuanYangEnv.transform(s, a)
a_num = self.find_anum(YuanYangEnv, a)
if t == True:
q_target = r
else:
# 下一个状态处的最大动作
a1 = self.epsilon_greedy_policy(YuanYangEnv, self.qvalue,
s_next, epsilon)
a1_num = self.find_anum(YuanYangEnv, a1)
# qlearning的更新公式
q_target = r + YuanYangEnv.gamma * self.qvalue[s_next,
a1_num]
# 利用td方法更新动作值函数
self.qvalue[s, a_num] = self.qvalue[
s, a_num] + alpha * (q_target - self.qvalue[s, a_num])
# YuanYangEnv2.bird_male_position = YuanYangEnv2.state_to_position(s)
# YuanYangEnv2.render()
# time.sleep(1)
# 转到下一个状态
s = s_next
a = self.epsilon_greedy_policy(YuanYangEnv, self.qvalue, s,
epsilon)
count += 1
return self.qvalue
#策略评估
v1 = r + self.gamma * self.v[s]
#策略改进
for action in self.actions:
s, r, t = yuanyang.transform( state, action )
if v1 < r + self.gamma * self.v[s]:
a1 = action
v1 = r + self.gamma * self.v[s]
delta+= abs(v1 - self.v[state])
self.pi[state] = a1
self.v[state] = v1
if delta < 1e-6:
print("迭代次数为",i)
break
if __name__ == "__main__":
yuanyang = YuanYangEnv()
policy_value = DP_Value_Iter(yuanyang)
policy_value.value_iteration()
# 将v值打印出来
s = 0
path = []
for state in range(100):
i = int(state / 10)
j = state % 10
yuanyang.value[j, i] = policy_value.v[state]
flag = 1
step_num = 0
# 将最优路径打印出来
while flag:
#渲染路径点
path.append(s)
action = self.pi[state]
s, r, t = yuanyang.transform(state, action)
new_v = r + self.gamma * self.v[s]
delta += abs(self.v[state] - new_v)
self.v[state] = new_v
if delta < 1e-6:
break
def policy_iterate(self):
for i in range(100):
self.policy_evaluate()
self.policy_improve()
if __name__ == "__main__":
yuanyang = YuanYangEnv()
policy_value = Policy_Value(yuanyang)
policy_value.policy_iterate()
flag = 1
s = 0
# print(policy_value.pi)
step_num = 0
#将最优路径打印出来
while flag:
a = policy_value.pi[s]
print('%d->%s\t' % (s, a))
yuanyang.bird_male_position = yuanyang.state_to_position(s)
yuanyang.render()
time.sleep(0.2)
step_num += 1
s_, r, t = yuanyang.transform(s, a)