def execute():
""" Execute Learning Algorithm """
global eligibility, q_old
assert initiated, "TOSL not initiated! setup() must be previously called"
s = lp.s
a = lp.a
alpha = lp.alpha
q = lp.q
v = lp.v
policy = lp.policy
# Specific Learning Algorithm
agent.execute_action(a)
time.sleep(lp.step_time)
# robot.stop()
# time.sleep(TASK.STEP_TIME/2)
sp = agent.observe_state()
r = agent.obtain_reward(s, a, sp)
ap = agent.select_action(sp) # Exploration strategy
diff_q = q[s, a] - q_old
q_old = q[sp, ap]
delta = r + exp.GAMMA * q[sp, ap] - q[s, a] # TD error
eligibility[s, a] = (1.0 - alpha) * eligibility[s, a] + 1
if eli_queue.count(s) > 0:
eli_queue.remove(s)
assert eli_queue.count(s) == 0, ("duplicated states found in ET ", str(s))
eli_queue.appendleft(s)
for i in eli_queue: # no all states updated, just those in eli_queue
# replace eli_queue by range(task.n_states) for non-reduced ET
for j in range(task.n_actions):
if eligibility[i, j] > 0.01:
q[i, j] = q[i, j] + alpha * (delta + diff_q) * eligibility[i, j]
eligibility[i, j] *= exp.GAMMA * exp.LAMBDA
else:
eligibility[i, j] = 0
if i == s and j == a:
q[i, j] = q[i, j] - alpha * diff_q
# update v and policy
v[i] = np.max(q[i])
policy[i] = np.argmax(q[i])
lp.s, lp.a = s, a
lp.sp, lp.ap = sp, ap
lp.r = r
lp.alpha = alpha
lp.q = q
lp.v = v
lp.policy = policy
lp.delta = delta
def execute():
"""Execute the learning algorithm"""
s = lp.s
a = lp.a
alpha = lp.alpha
q = lp.q
v = lp.v
policy = lp.policy
agent.execute_action(a)
time.sleep(lp.step_time)
sp = agent.observe_state()
r = agent.obtain_reward(s, a, sp)
ap = agent.select_action(sp)
# update Q
delta = r + exp.GAMMA * q[sp, ap] - q[s, a] # TD error (SARSA)
q[s, a] = q[s, a] + alpha * delta # update rule
# Update V and Policy
v[s] = np.max(q[s])
policy[s] = np.argmax(q[s])
lp.s = s
lp.a = a
lp.sp = sp
lp.ap = ap
lp.r = r
lp.alpha = alpha
lp.q = q
lp.v = v
lp.policy = policy
lp.delta = delta
def execute():
s=lp.s
q=lp.q
v=lp.v
policy = lp.policy
alpha = lp.alpha
gamma = lp.gamma
a=ap=agent.actionselection(s)
agent.execute_action(a)
time.sleep(lp.step_time)
sp=agent.observestate()
r=agent.get_reward()
q[s,a]=q[s,a]+alpha*(r+gamma*q[sp,ap]-q[s,a])
v[s]=np.max(q[s])
policy[s]=np.argmax(q[s])
lp.s = s
lp.a = a
lp.sp = sp
lp.ap = ap
lp.r = r
lp.q = q
lp.v = v
lp.policy = policy
return
def execute():
""" Execute the learning algorithm """
global eligibility
assert initiated, " SL not initiated! setup() must be previously called"
s = lp.s
a = lp.a
alpha = lp.alpha
q = lp.q
v = lp.v
policy = lp.policy
# Specific Learning Algorithm
agent.execute_action(a)
time.sleep(lp.step_time)
sp = agent.observe_state()
r = agent.obtain_reward(s, a, sp)
ap = agent.select_action(sp) # Exploration strategy
delta = r + exp.GAMMA * q[sp, ap] - q[s, a] # TD error
eligibility[s, a] = 1.0 # replace trace
if eli_queue.count(s) > 0:
eli_queue.remove(s)
assert eli_queue.count(s) == 0, ("duplicated states found in ET: ", str(s))
eli_queue.appendleft(s)
# only the states in eli_queue are updated:
for i in eli_queue: # no all states updated, just those in eli_queue
# replace eli_queue by range(task.n_states) for non-reduced ET
for j in range(task.n_actions):
if eligibility[i, j] > 0.01:
q[i, j] = q[i, j] + alpha * delta * eligibility[i, j]
eligibility[i, j] *= exp.GAMMA * exp.LAMBDA
else:
eligibility[i, j] = 0
# update v and policy
v[i] = np.max(q[i])
policy[i] = np.argmax(q[i])
lp.s = s
lp.a = a
lp.sp = sp
lp.ap = ap
lp.r = r
lp.alpha = alpha
lp.q = q
lp.v = v
lp.policy = policy
lp.delta = delta
def run_DQN():
import agent
import time
import exp
import rl_vrep
rl_vrep.connect()
rl_vrep.start()
time.sleep(0.5)
agent.setup_task()
time.sleep(0.5)
agent.setup()
time.sleep(0.5)
step = 0
for epi in range(30000):
print('step:', step)
observation = np.empty(shape=(10, 10))
observation_ = np.empty(shape=(10, 10))
for i in range(10):
observation_i = agent.observestate()
observation_i = agent.unwrap_state(observation_i)
observation[i] = np.array(observation_i)
observation = observation[np.newaxis, :]
observation = Robot.sess.run(Robot.rnn_out,
feed_dict={Robot.rnn_in: observation})
#observation = np.mean(observation,axis= 0)
action = Robot.choose_action(observation)
agent.execute_action(action)
print('action:', action)
#time.sleep(0.5)
r = agent.get_reward()
print('reward:', r)
for i in range(10):
observation_i = agent.observestate()
observation_i = agent.unwrap_state(observation_i)
observation_[i] = np.array(observation_i)
#observation_ = np.mean(observation_,axis= 0)
#.........
observation_ = observation_[np.newaxis, :]
observation_ = Robot.sess.run(Robot.rnn_out,
feed_dict={Robot.rnn_in: observation_})
Robot.store(observation, action, r, observation_)
if (step > 200) and (step % 10 == 0):
Robot.learn()
observation = observation_
step += 1
print('run over!')
def work(self, sess, coord):
print('starting %s...\n' % self.name)
while not coord.should_stop():
sess.run(self.update_local_op)
memory = []
s = agent.observestate()
s = agent.unwrap_state(s)
while True:
p, v = sess.run([self.local_net.policy, self.local_net.value],
feed_dict={self.local_net.s: s[np.newaxis, :]})
a = np.random.choice(range(N_A), p=p[0])
agent.execute_action(a)
print('action:', a)
time.sleep(0.3)
r = agent.get_reward()
s1 = agent.observestate()
s1 = agent.unwrap_state(s1)
memory.append([s, a, r, s1, v[0][0]])
s = s1
global_steps = next(self.global_steps_counter)
self.local_steps += 1
#sess.run(self.anneal_lr_op)
collide = None
if np.where(s1 < 0):
collide = True
else:
collide = False
if not collide and len(memory) == MEMORY_SIZE:
v1 = sess.run(self.local_net.value,
feed_dict={self.local_net.s: s})
self.train(memory, sess, v1[0][0], global_steps)
memory = []
sess.run(self.update_local_op)
if collide:
break
if len(memory) != 0:
self.train(memory, sess, 0.0, global_steps)
if global_steps >= MAX_STEP:
COORD.request_stop()
"""
rl_vrep.connect()
rl_vrep.start()
time.sleep(0.5)
agent.setup_task()
time.sleep(0.5)
agent.setup()
#robot.setup(["MOBILE_BASE", "DISTANCE_SENSOR"],[])
time.sleep(0.5)
policy = np.zeros((16384, 1), dtype=np.int)
with open("policy_results.txt", 'r') as f:
for line in f:
line = line.strip()
if len(line) == 0: continue
results = line.split(":")
policy[int(results[0])] = int(results[1])
learning_process.policy = policy
print('policy:')
print(learning_process.policy)
i = 0
while 1:
s = agent.observestate()
print('%d-s:' % (i), s)
a = learning_process.policy[s]
print('%d-a:' % (i), a)
agent.execute_action(a)
time.sleep(0.5)
i += 1