for t in xrange(1, STEPS + 1): if DISPLAY: env.render() at = Actor.predict(st) + n.Sample() # execute action st_next, rt, Done, _ = env.step(at[0]) mdp.add_frame(st_next) st_next = mdp.get_MDP_state() if Done: dt = 1 else: dt = 0 totalR += rt # store transition R.StoreTransition(st, at, np.array([rt]), st_next, dt) st = st_next if episode_i > OBSERVATION_PHASE: for mini_batch in xrange(BATCHES): # sample mini batch s_batch, a_batch, r_batch, stag_batch, terminal_batch, _ = R.SampleMiniBatch( MINI_BATCH) Q_next = Critic.target_predict( stag_batch, Actor.target_predict(stag_batch)) Y = r_batch + GAMMA * Q_next * (1 - terminal_batch) Critic.train(Y, s_batch, a_batch) a_for_grad = Actor.predict(s_batch)
if EXP_PROB > EPSILON: EXP_PROB -= ann_fric # execute action st_next, rt, Done, _ = env.step(at) mdp.add_frame(st_next) rt = rt+T st_next = mdp.get_MDP_state() if Done: dt = 1 else: dt = 0 totalR += rt # store transition R.StoreTransition(st, np.array([a_index]), np.array([rt]), st_next, dt) st = st_next if episode_i > OBSERVATION_PHASE: for mini_batch in xrange(BATCHES): # sample mini batch s_batch, a_batch, r_batch, stag_batch, terminal_batch = R.SampleMiniBatch(MINI_BATCH) Y = Q.evaluate(sess, s_batch) Q_next = Q_target.evaluate(sess, stag_batch) Q_next_max = np.amax(Q_next,1) a_batch = a_batch.astype(int) for i in range(MINI_BATCH): Y[i,a_batch[i,0]] = r_batch[i,0] + GAMMA*Q_next_max[i] * (1-terminal_batch[i])