예제 #1
0
    def playActor(self):
        self.load("NetworkParam/FinalParam")

        hdg0_rand_vec=[0,7,12]
        '''
        WIND CONDITIONS
        '''
        mean = 45 * TORAD
        std = 0.1 * TORAD
        wind_samples = 10
        w = wind(mean=mean, std=std, samples = wind_samples)

        try:
            for i in range(len(hdg0_rand_vec)):
                # Initial state
                WH = w.generateWind()
                hdg0_rand = hdg0_rand_vec[i]
                hdg0 = hdg0_rand * TORAD * np.ones(10)
                s = self.env.reset(hdg0,WH)
                episode_reward = 0
                episode_step=0
                v_episode=[]
                i_episode=[]
                while episode_step < 40: #not done:
                    if episode_step==0:
                        i_episode.append(hdg0_rand+WH[0]/TORAD-40)
                    else:
                        i_episode.append(s[0][-1]/TORAD)
                    s = np.reshape([s[0,:], s[1,:]], [self.state_size,1])

                    a, = self.sess.run(self.network.actions,
                                       feed_dict={self.network.state_ph: s[None]})
                    a = np.clip(a, self.low_bound, self.high_bound)
                    s_, r   = self.env.act(a,WH)
                    episode_reward += r
                    v_episode.append(r)
                    episode_step += 1
                    s = s_
                DISPLAYER.displayVI(v_episode,i_episode,i)
                print("Episode reward :", episode_reward," for incidence: ",hdg0_rand)

        except KeyboardInterrupt as e:
            pass

        except Exception as e:
            print("Exception :", e)

        finally:
            print("End of the demo")
예제 #2
0
    def __init__(self, sess):
        print("Initializing the agent...")

        self.sess = sess
        self.env = Environment()
        self.state_size = self.env.get_state_size()[0]
        self.action_size = self.env.get_action_size()
        self.low_bound, self.high_bound = self.env.get_bounds()

        self.buffer = ExperienceBuffer()

        print("Creation of the actor-critic network")
        self.network = Network(self.state_size, self.action_size,
                               self.low_bound, self.high_bound)

        self.sess.run(tf.global_variables_initializer())
        DISPLAYER.reset()
예제 #3
0
    def run(self):

        self.total_steps = 1
        self.sess.run(self.network.target_init)
        self.z = self.sess.run(self.network.z)
        self.delta_z = self.network.delta_z

        ep = 1
        while ep < settings.TRAINING_EPS + 1 and not GUI.STOP:

            s = self.env.reset()
            episode_reward = 0
            episode_step = 0
            done = False
            memory = deque()

            # Initialize exploration noise process
            noise_scale = settings.NOISE_SCALE * settings.NOISE_DECAY**ep

            # Initial state
            self.env.set_render(GUI.render.get(ep))
            self.env.set_gif(GUI.gif.get(ep))
            plot_distrib = GUI.plot_distrib.get(ep)

            max_eps = settings.MAX_EPISODE_STEPS + (ep // 50)

            while episode_step < max_eps and not done:

                noise = np.random.normal(size=self.action_size)
                scaled_noise = noise_scale * noise

                a = np.clip(
                    self.predict_action(s, plot_distrib) + scaled_noise,
                    *self.bounds)

                s_, r, done, info = self.env.act(a)

                episode_reward += r

                memory.append((s, a, r, s_, 0 if done else 1))

                if len(memory) >= settings.N_STEP_RETURN:
                    s_mem, a_mem, discount_r, ss_mem, done_mem = memory.popleft(
                    )
                    for i, (si, ai, ri, s_i, di) in enumerate(memory):
                        discount_r += ri * settings.DISCOUNT**(i + 1)
                    BUFFER.add(s_mem, a_mem, discount_r, s_, 0 if done else 1)

                if len(
                        BUFFER
                ) > 0 and self.total_steps % settings.TRAINING_FREQ == 0:
                    self.network.train(BUFFER.sample(), self.critic_lr,
                                       self.actor_lr)

                s = s_
                episode_step += 1
                self.total_steps += 1

            self.critic_lr -= self.delta_critic_lr
            self.actor_lr -= self.delta_actor_lr

            # Plot reward
            plot = GUI.plot.get(ep)
            DISPLAYER.add_reward(episode_reward, plot)

            # Print episode reward
            if GUI.ep_reward.get(ep):
                print(
                    'Episode %2i, Reward: %7.3f, Steps: %i, Final noise scale: %7.3f, Critic LR: %f, Actor LR: %f'
                    % (ep, episode_reward, episode_step, noise_scale,
                       self.critic_lr, self.actor_lr))

            # Save the model
            if GUI.save.get(ep):
                SAVER.save(ep)

            ep += 1
예제 #4
0
    def run(self):
        print("Beginning of the run...")

        self.pre_train()

        self.total_steps = 0
        self.nb_ep = 1

        while self.nb_ep < parameters.TRAINING_STEPS:

            self.learning_rate = self.initial_learning_rate * \
                (parameters.TRAINING_STEPS - self.nb_ep) / \
                parameters.TRAINING_STEPS

            s = self.env.reset()
            episode_reward = 0
            done = False

            memory = deque()
            discount_R = 0

            episode_step = 0
            max_step = parameters.MAX_EPISODE_STEPS + \
                self.nb_ep // parameters.EP_ELONGATION

            # Render parameters
            self.env.set_render(self.nb_ep % parameters.RENDER_FREQ == 0)

            while episode_step < max_step and not done:

                if random.random() < self.epsilon:
                    a = random.randint(0, self.action_size - 1)
                else:
                    a = self.sess.run(self.mainQNetwork.predict,
                                      feed_dict={self.mainQNetwork.inputs: [s]})
                    a = a[0]

                s_, r, done, info = self.env.act(a)
                episode_reward += r

                memory.append((s, a, r, s_, done))

                if len(memory) > parameters.N_STEP_RETURN:
                    s_mem, a_mem, r_mem, ss_mem, done_mem = memory.popleft()
                    discount_R = r_mem
                    for i, (si, ai, ri, s_i, di) in enumerate(memory):
                        discount_R += ri * parameters.DISCOUNT ** (i + 1)
                    self.buffer.add(s_mem, a_mem, discount_R, s_, done)

                if episode_step % parameters.TRAINING_FREQ == 0:

                    train_batch = self.buffer.sample(parameters.BATCH_SIZE,
                                                     self.beta)
                    # Incr beta
                    if self.beta <= parameters.BETA_STOP:
                        self.beta += parameters.BETA_INCR

                    feed_dict = {self.mainQNetwork.inputs: train_batch[0]}
                    oldQvalues = self.sess.run(self.mainQNetwork.Qvalues,
                                               feed_dict=feed_dict)
                    tmp = [0] * len(oldQvalues)
                    for i, oldQvalue in enumerate(oldQvalues):
                        tmp[i] = oldQvalue[train_batch[1][i]]
                    oldQvalues = tmp

                    feed_dict = {self.mainQNetwork.inputs: train_batch[3]}
                    mainQaction = self.sess.run(self.mainQNetwork.predict,
                                                feed_dict=feed_dict)

                    feed_dict = {self.targetQNetwork.inputs: train_batch[3]}
                    targetQvalues = self.sess.run(self.targetQNetwork.Qvalues,
                                                  feed_dict=feed_dict)

                    # Done multiplier :
                    # equals 0 if the episode was done
                    # equals 1 else
                    done_multiplier = (1 - train_batch[4])
                    doubleQ = targetQvalues[range(parameters.BATCH_SIZE),
                                            mainQaction]
                    targetQvalues = train_batch[2] + \
                        parameters.DISCOUNT * doubleQ * done_multiplier

                    errors = np.square(targetQvalues - oldQvalues) + 1e-6
                    self.buffer.update_priorities(train_batch[6], errors)

                    feed_dict = {self.mainQNetwork.inputs: train_batch[0],
                                 self.mainQNetwork.Qtarget: targetQvalues,
                                 self.mainQNetwork.actions: train_batch[1],
                                 self.mainQNetwork.learning_rate: self.learning_rate}
                    _ = self.sess.run(self.mainQNetwork.train,
                                      feed_dict=feed_dict)

                    update_target(self.update_target_ops, self.sess)

                s = s_
                episode_step += 1
                self.total_steps += 1

            # Decay epsilon
            if self.epsilon > parameters.EPSILON_STOP:
                self.epsilon -= parameters.EPSILON_DECAY

            DISPLAYER.add_reward(episode_reward)
            # if episode_reward > self.best_run and \
            #         self.nb_ep > 50:
            #     self.best_run = episode_reward
            #     print("Save best", episode_reward)
            #     SAVER.save('best')
            #     self.play(1)

            self.total_steps += 1

            if self.nb_ep % parameters.DISP_EP_REWARD_FREQ == 0:
                print('Episode %2i, Reward: %7.3f, Steps: %i, Epsilon: %.3f'
                      ', Max steps: %i, Learning rate: %g' % (
                          self.nb_ep, episode_reward, episode_step,
                          self.epsilon, max_step, self.learning_rate))

            # Save the model
            if self.nb_ep % parameters.SAVE_FREQ == 0:
                SAVER.save(self.nb_ep)

            self.nb_ep += 1
예제 #5
0
from Displayer import DISPLAYER
from Saver import SAVER

import parameters

if __name__ == '__main__':

    tf.reset_default_graph()

    with tf.Session() as sess:

        agent = Agent(sess)
        SAVER.set_sess(sess)

        SAVER.load(agent)

        print("Beginning of the run")
        try:
            agent.run()
        except KeyboardInterrupt:
            pass
        print("End of the run")
        SAVER.save(agent.total_steps)
        DISPLAYER.disp()

        # agent.play(10)

        # agent.play(3, "results/gif/".format(parameters.ENV))

    agent.close()
예제 #6
0
    def run(self):

        self.total_steps = 0

        for ep in range(1, parameters.TRAINING_STEPS + 1):

            episode_reward = 0
            episode_step = 0
            done = False

            # Initial state
            s = self.env.reset()
            self.env.set_render(ep % 1000 == 0)
            gif = (ep % 1500 == 0)
            step_allonge = ep // 1000

            while episode_step < parameters.MAX_EPISODE_STEPS + step_allonge \
                    and not done:

                if random.random() < self.epsilon:
                    a = self.env.random()
                else:
                    # choose action based on deterministic policy
                    a, = self.sess.run(self.network.actions,
                                       feed_dict={self.network.state_ph: [s]})

                s_, r, done, info = self.env.act(a, gif)
                episode_reward += r

                self.buffer.add((s, a, r, s_, 0.0 if done else 1.0))

                # update network weights to fit a minibatch of experience
                if self.total_steps % parameters.TRAINING_FREQ == 0 and \
                        len(self.buffer) >= parameters.BATCH_SIZE:

                    minibatch = self.buffer.sample()

                    _, _ = self.sess.run([self.network.critic_train_op, self.network.actor_train_op],
                                         feed_dict={
                        self.network.state_ph: np.asarray([elem[0] for elem in minibatch]),
                        self.network.action_ph: np.asarray([elem[1] for elem in minibatch]),
                        self.network.reward_ph: np.asarray([elem[2] for elem in minibatch]),
                        self.network.next_state_ph: np.asarray([elem[3] for elem in minibatch]),
                        self.network.is_not_terminal_ph: np.asarray([elem[4] for elem in minibatch])})

                    # update target networks
                    _ = self.sess.run(self.network.update_slow_targets_op)

                s = s_
                episode_step += 1
                self.total_steps += 1

            # Decay epsilon
            if self.epsilon > parameters.EPSILON_STOP:
                self.epsilon -= self.epsilon_decay

            if gif:
                self.env.save_gif('results/gif/', self.n_gif)
                self.n_gif = (self.n_gif + 1) % 5

            if episode_reward > self.best_run:
                self.best_run = episode_reward
                print("Save best", episode_reward)
                SAVER.save('best')

            DISPLAYER.add_reward(episode_reward)
            if ep % 50 == 0:
                print('Episode %2i, Reward: %7.3f, Steps: %i, Epsilon: %7.3f'
                      ' (max step: %i)' % (ep, episode_reward, episode_step,
                                           self.epsilon,
                                           parameters.MAX_EPISODE_STEPS +
                                           step_allonge))
            if ep % 500 == 0:
                DISPLAYER.disp()
예제 #7
0
        SAVER.set_sess(sess)

        SAVER.load()

        # Run threads that each contains one worker
        worker_threads = []
        for i, worker in enumerate(workers):
            print("Threading worker", i + 1)
            sleep(0.05)
            work = lambda: worker.work(sess, coord)
            t = threading.Thread(target=(work))
            t.start()
            worker_threads.append(t)
            sleep(0.1)

        try:
            # Wait till all the workers are done
            coord.join(worker_threads)
        except Exception as e:
            coord.request_stop(e)
        except KeyboardInterrupt as e:
            coord.request_stop()
        finally:
            print("End of the training")

        DISPLAYER.disp_all()
        DISPLAYER.disp_one()
        DISPLAYER.disp_seq()
        master_agent.play(sess, 10)
        master_agent.play(sess, 1, "results/gif/{}_1.gif".format(settings.ENV))
예제 #8
0
파일: main.py 프로젝트: guillemaru/IBOAT_RL
import tensorflow as tf

from Agent import Agent

from Displayer import DISPLAYER

import parameters

if __name__ == '__main__':
    
    tf.reset_default_graph()

    with tf.Session() as sess:

        agent = Agent(sess)
        
        print("Beginning of the run")
        try:
            print("I do not do the run")
            #agent.run()
        except KeyboardInterrupt:
            agent.save("NetworkParam/FinalParam")
        print("End of the run")
        DISPLAYER.dispR()
	    
        agent.playActor()
        agent.playCritic()
예제 #9
0
    def playCritic(self):
        self.load("NetworkParam/FinalParam")

        hdg0_rand_vec=[0,7,12]
        '''
        WIND CONDITIONS
        '''
        mean = 45 * TORAD
        std = 0.1 * TORAD
        wind_samples = 10
        w = wind(mean=mean, std=std, samples = wind_samples)

        try:
            for i in range(len(hdg0_rand_vec)):
                # Initial state
                WH = w.generateWind()
                hdg0_rand = hdg0_rand_vec[i]
                hdg0 = hdg0_rand * TORAD * np.ones(10)
                s = self.env.reset(hdg0,WH)
                
                episode_reward = 0
                episode_step=0
                v_episode=[]
                i_episode=[]
                while episode_step < 30: #not done:
                    if episode_step==0:
                        i_episode.append(hdg0_rand+WH[0]/TORAD-40)
                    else:
                        i_episode.append(s[0][-1]/TORAD)
                    
                    # Critic policy
                    critic = [self.evaluate(s, -1.5),self.evaluate(s, -1.25),self.evaluate(s, -1),
                        self.evaluate(s, -0.75),self.evaluate(s, -0.5),self.evaluate(s, -0.25),self.evaluate(s, 0),self.evaluate(s, 0.25),
                            self.evaluate(s, 0.5),self.evaluate(s, 0.75),self.evaluate(s, 1),self.evaluate(s, 1.25),
                            self.evaluate(s, 1.5)]
                    a = np.argmax(critic)
                    if a == 0:
                        a = -1.5
                    if a == 1:
                        a = -1.25
                    if a == 2:
                        a = -1
                    if a == 3:
                        a = -0.75
                    if a == 4:
                        a = -0.5
                    if a == 5:
                        a = -0.25
                    if a == 6:
                        a = 0
                    if a == 7:
                        a = 0.25
                    if a == 8:
                        a = 0.5
                    if a == 9:
                        a = 0.75
                    if a == 10:
                        a = 1
                    if a == 11:
                        a = 1.25
                    if a == 12:
                        a = 1.5

                    s_, r   = self.env.act(a,WH)
                    episode_reward += r
                    v_episode.append(r)
                    episode_step += 1
                    s = s_
                DISPLAYER.displayVI(v_episode,i_episode,i+3)
                print("Episode reward :", episode_reward," for incidence: ",hdg0_rand)

        except KeyboardInterrupt as e:
            pass

        except Exception as e:
            print("Exception :", e)

        finally:
            print("End of the demo")
예제 #10
0
    def work(self, sess, coord):
        print("Running", self.name, end='\n\n')
        self.starting_time = time()
        self.nb_ep = 1
        nearlyDone = 0
        with sess.as_default(), sess.graph.as_default():

            with coord.stop_on_exception():
                while not coord.should_stop():

                    self.states_buffer = []
                    self.actions_buffer = []
                    self.rewards_buffer = []
                    self.values_buffer = []
                    self.mean_values_buffer = []

                    self.total_steps = 0
                    episode_reward = 0
                    episode_step = 0

                    # Reset the local network to the global
                    sess.run(self.update_local_vars)

                    mean = 45 * TORAD
                    std = 0 * TORAD
                    wind_samples = 10
                    w = wind(mean=mean, std=std, samples=wind_samples)
                    WH = w.generateWind()
                    hdg0_rand = random.uniform(5, 12)
                    hdg0 = hdg0_rand * TORAD * np.ones(10)
                    s = self.env.reset(hdg0, WH)

                    done = False
                    #if self.worker_index == 1 and render and settings.DISPLAY:
                    #    self.env.set_render(True)

                    #self.lstm_state = self.network.lstm_state_init
                    #self.initial_lstm_state = self.lstm_state

                    while not coord.should_stop() and not done and \
                            episode_step < settings.MAX_EPISODE_STEP:

                        WH = np.random.uniform(mean - std,
                                               mean + std,
                                               size=wind_samples)
                        s = np.reshape([s[0, :], s[1, :]],
                                       [2 * self.state_size, 1])

                        # Prediction of the policy and the value
                        feed_dict = {self.network.inputs: [s]}
                        policy, value = sess.run(
                            [self.network.policy, self.network.value],
                            feed_dict=feed_dict)

                        policy, value = policy[0], value[0][0]

                        if random.random() < self.epsilon:
                            action = random.choice([1.5, 0, -1.5])

                        else:
                            # Choose an action according to the policy
                            action = np.random.choice([1.5, 0, -1.5], p=policy)

                        s_, v = self.env.act(action, WH)

                        #reward  assignation algorithm
                        if episode_step == 1:
                            r = 0
                        elif s[int(self.state_size / 2 - 2)] > (
                                13 *
                                TORAD) and s[int(self.state_size / 2 - 2)] < (
                                    15 * TORAD
                                ) and v > 0.63 and v < 0.67 and action < 0:
                            r = 0.5
                        else:
                            if v <= 0.69:
                                r = 0
                                nearlyDone = 0
                            elif v > 0.69 and v <= 0.75:
                                r = 0.00001
                                nearlyDone = 0
                            elif v > 0.75 and v <= 0.8:
                                r = 0.01
                                nearlyDone = 0
                            elif v > 0.80:
                                r = 0.1
                                if nearlyDone >= 3:
                                    r = 1
                                    done = True
                                elif nearlyDone == 2:
                                    r = 0.8
                                elif nearlyDone == 1:
                                    r = 0.25
                                nearlyDone = nearlyDone + 1
                            else:
                                r = 0
                                nearlyDone = False

                        #s_ = np.reshape(s_, [2*self.state_size,1])

                        # Store the experience
                        self.states_buffer.append(s)
                        self.actions_buffer.append(action)
                        self.rewards_buffer.append(r)
                        self.values_buffer.append(value)
                        self.mean_values_buffer.append(value)
                        episode_reward += r
                        s = s_

                        episode_step += 1
                        self.total_steps += 1

                        # If we have more than MAX_LEN_BUFFER experiences, we
                        # apply the gradients and update the global network,
                        # then we empty the episode buffers
                        if len(self.states_buffer) == settings.MAX_LEN_BUFFER \
                                and not done:

                            feed_dict = {
                                self.network.inputs: [
                                    np.reshape([s[0, :], s[1, :]],
                                               [2 * self.state_size, 1])
                                ]
                            }
                            bootstrap_value = sess.run(self.network.value,
                                                       feed_dict=feed_dict)

                            self.train(sess, bootstrap_value
                                       )  #with this we change global network
                            sess.run(self.update_local_vars)
                            #self.initial_lstm_state = self.lstm_state

                    if len(self.states_buffer) != 0:
                        if done:
                            bootstrap_value = 0
                        else:
                            feed_dict = {
                                self.network.inputs: [
                                    np.reshape([s[0, :], s[1, :]],
                                               [2 * self.state_size, 1])
                                ]
                            }
                            bootstrap_value = sess.run(self.network.value,
                                                       feed_dict=feed_dict)
                        self.train(sess, bootstrap_value)

                    if self.epsilon > settings.EPSILON_STOP:
                        self.epsilon -= settings.EPSILON_DECAY

                    self.nb_ep += 1

                    if not coord.should_stop():
                        DISPLAYER.add_reward(episode_reward, self.worker_index)

                    if (self.worker_index == 1 and
                            self.nb_ep % settings.DISP_EP_REWARD_FREQ == 0):
                        print(
                            'Episode %2i, Initial hdg: %2i, Reward: %7.3f, Steps: %i, '
                            'Epsilon: %7.3f' %
                            (self.nb_ep, hdg0_rand, episode_reward,
                             episode_step, self.epsilon))
                        print("Policy: ", policy)
                    if (self.worker_index == 1
                            and self.nb_ep % settings.SAVE_FREQ == 0):
                        self.save(self.total_steps)

                    if time() - self.starting_time > settings.LIMIT_RUN_TIME:
                        coord.request_stop()

            self.summary_writer.close()
예제 #11
0
    def run(self):

        self.total_steps = 0

        for ep in range(1, parameters.TRAINING_STEPS + 1):

            episode_reward = 0
            episode_step = 0
            done = False

            # Initialize exploration noise process
            noise_process = np.zeros(self.action_size)
            noise_scale = (parameters.NOISE_SCALE_INIT *
                           parameters.NOISE_DECAY**ep) * \
                (self.high_bound - self.low_bound)

            # Initial state
            s = self.env.reset()
            render = (ep % parameters.RENDER_FREQ == 0 and parameters.DISPLAY)
            self.env.set_render(render)

            while episode_step < parameters.MAX_EPISODE_STEPS and not done:

                # choose action based on deterministic policy
                a, = self.sess.run(self.network.actions,
                                   feed_dict={self.network.state_ph: s[None]})

                # add temporally-correlated exploration noise to action
                # (using an Ornstein-Uhlenbeck process)
                noise_process = parameters.EXPLO_THETA * \
                    (parameters.EXPLO_MU - noise_process) + \
                    parameters.EXPLO_SIGMA * np.random.randn(self.action_size)

                a += noise_scale * noise_process

                s_, r, done, info = self.env.act(a)
                episode_reward += r

                self.buffer.add((s, a, r, s_, 0.0 if done else 1.0))

                # update network weights to fit a minibatch of experience
                if self.total_steps % parameters.TRAINING_FREQ == 0 and \
                        len(self.buffer) >= parameters.BATCH_SIZE:

                    minibatch = self.buffer.sample()

                    _, _ = self.sess.run(
                        [
                            self.network.critic_train_op,
                            self.network.actor_train_op
                        ],
                        feed_dict={
                            self.network.state_ph:
                            np.asarray([elem[0] for elem in minibatch]),
                            self.network.action_ph:
                            np.asarray([elem[1] for elem in minibatch]),
                            self.network.reward_ph:
                            np.asarray([elem[2] for elem in minibatch]),
                            self.network.next_state_ph:
                            np.asarray([elem[3] for elem in minibatch]),
                            self.network.is_not_terminal_ph:
                            np.asarray([elem[4] for elem in minibatch])
                        })

                    # update target networks
                    _ = self.sess.run(self.network.update_slow_targets_op)

                s = s_
                episode_step += 1
                self.total_steps += 1

            if ep % parameters.DISP_EP_REWARD_FREQ == 0:
                print(
                    'Episode %2i, Reward: %7.3f, Steps: %i, Final noise scale: %7.3f'
                    % (ep, episode_reward, episode_step, noise_scale))
            DISPLAYER.add_reward(episode_reward)
예제 #12
0
    def run(self):
        print("Beginning of the run...")

        self.pre_train()

        self.total_steps = 0
        self.nb_ep = 1

        while self.nb_ep < parameters.TRAINING_STEPS:

            s = self.env.reset()
            episode_reward = 0
            done = False

            memory = deque()
            discount_R = 0

            episode_step = 0

            # Render parameters
            self.env.set_render(self.nb_ep % parameters.RENDER_FREQ == 0)

            while episode_step < parameters.MAX_EPISODE_STEPS and not done:

                if random.random() < self.epsilon:
                    a = random.randint(0, self.action_size - 1)
                else:
                    a = self.sess.run(
                        self.mainQNetwork.predict,
                        feed_dict={self.mainQNetwork.inputs: [s]})
                    a = a[0]

                s_, r, done, info = self.env.act(a)
                episode_reward += r

                memory.append((s, a, r, s_, done))

                if len(memory) > parameters.N_STEP_RETURN:
                    s_mem, a_mem, r_mem, ss_mem, done_mem = memory.popleft()
                    discount_R = r_mem
                    for i, (si, ai, ri, s_i, di) in enumerate(memory):
                        discount_R += ri * parameters.DISCOUNT**(i + 1)
                    self.buffer.add(s_mem, a_mem, discount_R, s_, done)

                if episode_step % parameters.TRAINING_FREQ == 0:

                    train_batch = self.buffer.sample(parameters.BATCH_SIZE,
                                                     self.beta)
                    # Incr beta
                    if self.beta <= parameters.BETA_STOP:
                        self.beta += parameters.BETA_INCR

                    feed_dict = {self.mainQNetwork.inputs: train_batch[3]}
                    mainQaction = self.sess.run(self.mainQNetwork.predict,
                                                feed_dict=feed_dict)

                    feed_dict = {self.targetQNetwork.inputs: train_batch[3]}
                    targetQvalues = self.sess.run(self.targetQNetwork.Qvalues,
                                                  feed_dict=feed_dict)

                    # Done multiplier :
                    # equals 0 if the episode was done
                    # equals 1 else
                    done_multiplier = (1 - train_batch[4])
                    doubleQ = targetQvalues[range(parameters.BATCH_SIZE),
                                            mainQaction]
                    targetQvalues = train_batch[2] + \
                        parameters.DISCOUNT * doubleQ * done_multiplier

                    feed_dict = {
                        self.mainQNetwork.inputs: train_batch[0],
                        self.mainQNetwork.Qtarget: targetQvalues,
                        self.mainQNetwork.actions: train_batch[1]
                    }
                    td_error, _ = self.sess.run(
                        [self.mainQNetwork.td_error, self.mainQNetwork.train],
                        feed_dict=feed_dict)

                    self.buffer.update_priorities(train_batch[6],
                                                  td_error + 1e-6)

                    update_target(self.update_target_ops, self.sess)

                s = s_
                episode_step += 1
                self.total_steps += 1

            # Decay epsilon
            if self.epsilon > parameters.EPSILON_STOP:
                self.epsilon -= parameters.EPSILON_DECAY

            DISPLAYER.add_reward(episode_reward)

            self.total_steps += 1

            if self.nb_ep % parameters.DISP_EP_REWARD_FREQ == 0:
                print('Episode %2i, Reward: %7.3f, Steps: %i, Epsilon: %f' %
                      (self.nb_ep, episode_reward, episode_step, self.epsilon))
            self.nb_ep += 1
예제 #13
0
    def work(self, sess, coord):
        print("Running", self.name, end='\n\n')
        self.starting_time = time()
        self.nb_ep = 1

        with sess.as_default(), sess.graph.as_default():

            with coord.stop_on_exception():
                while not coord.should_stop():

                    self.states_buffer = []
                    self.actions_buffer = []
                    self.rewards_buffer = []
                    self.values_buffer = []
                    self.mean_values_buffer = []
                    self.lstm_buffer = []

                    self.total_steps = 0
                    episode_reward = 0
                    episode_step = 0

                    # Reset the local network to the global
                    sess.run(self.update_local_vars)

                    s = self.env.reset()
                    done = False
                    render = (self.nb_ep % parameters.RENDER_FREQ == 0)
                    if render and parameters.DISPLAY:
                        self.env.set_render(True)

                    self.lstm_state = self.network.lstm_state_init
                    self.initial_lstm_state = self.lstm_state

                    while not coord.should_stop() and not done and \
                            episode_step < parameters.MAX_EPISODE_STEP:

                        self.lstm_buffer.append(self.lstm_state)

                        # Prediction of the policy and the value
                        feed_dict = {
                            self.network.inputs: [s],
                            self.network.state_in: self.lstm_state
                        }
                        policy, value, self.lstm_state = sess.run(
                            [
                                self.network.policy, self.network.value,
                                self.network.state_out
                            ],
                            feed_dict=feed_dict)

                        policy, value = policy[0], value[0][0]

                        if random.random() < self.epsilon:
                            action = random.randint(0, self.action_size - 1)

                        else:
                            # Choose an action according to the policy
                            action = np.random.choice(self.action_size,
                                                      p=policy)

                        s_, r, done, _ = self.env.act(action)

                        # Store the experience
                        self.states_buffer.append(s)
                        self.actions_buffer.append(action)
                        self.rewards_buffer.append(r)
                        self.values_buffer.append(value)
                        self.mean_values_buffer.append(value)
                        episode_reward += r
                        s = s_

                        episode_step += 1
                        self.total_steps += 1

                        # If we have more than MAX_LEN_BUFFER experiences, we
                        # apply the gradients and update the global network,
                        # then we empty the episode buffers
                        if len(self.states_buffer) == parameters.MAX_LEN_BUFFER \
                                and not done:

                            feed_dict = {
                                self.network.inputs: [s],
                                self.network.state_in: self.lstm_state
                            }
                            bootstrap_value = sess.run(self.network.value,
                                                       feed_dict=feed_dict)

                            self.train(sess, bootstrap_value)
                            sess.run(self.update_local_vars)
                            self.initial_lstm_state = self.lstm_state

                    if len(self.states_buffer) != 0:
                        if done:
                            bootstrap_value = 0
                        else:
                            feed_dict = {
                                self.network.inputs: [s],
                                self.network.state_in: self.lstm_state
                            }
                            bootstrap_value = sess.run(self.network.value,
                                                       feed_dict=feed_dict)
                        self.train(sess, bootstrap_value)

                    if self.epsilon > parameters.EPSILON_STOP:
                        self.epsilon -= parameters.EPSILON_DECAY

                    self.nb_ep += 1

                    if not coord.should_stop():
                        DISPLAYER.add_reward(episode_reward, self.worker_index)

                    if self.nb_ep % parameters.DISP_EP_REWARD_FREQ == 0:
                        print('Agent: %i, Episode %2i, Reward: %i, Steps: %i, '
                              'Epsilon: %7.3f' %
                              (self.worker_index, self.nb_ep, episode_reward,
                               episode_step, self.epsilon))

                    if (self.worker_index == 1
                            and self.nb_ep % parameters.SAVE_FREQ == 0):
                        self.save(self.total_steps)

                    if time() - self.starting_time > parameters.LIMIT_RUN_TIME:
                        coord.request_stop()

                    self.env.set_render(False)

            self.summary_writer.close()
            self.env.close()
예제 #14
0
    def run(self):
        #self.load("NetworkParam_best_ThirdSemester/FinalParam") #get the best parameters to start the training
        self.total_steps = 0

        '''
        WIND CONDITIONS
        '''
        mean = 45 * TORAD
        std = 0.1 * TORAD
        wind_samples = 10
        w = wind(mean=mean, std=std, samples = wind_samples)
        WH = w.generateWind()

        for ep in range(1, parameters.TRAINING_STEPS+1):

            episode_reward = 0
            episode_step = 0
            nearlyDone=0
            done=False

            # Initialize exploration noise process
            noise_process = np.zeros(self.action_size)
            noise_scale = (parameters.NOISE_SCALE_INIT *
                           parameters.NOISE_DECAY**ep) * \
                (self.high_bound - self.low_bound)

            # Initial state
            w = wind(mean=mean, std=std, samples = wind_samples)
            WH = w.generateWind()
            hdg0_rand = random.uniform(6,13) 
            hdg0 = hdg0_rand * TORAD * np.ones(10)
            s = self.env.reset(hdg0,WH)
            
            while episode_step < parameters.MAX_EPISODE_STEPS: #and not done:

                WH = np.random.uniform(mean - std, mean + std, size=wind_samples)

                # choose action based on deterministic policy
                s = np.reshape([s[0,:], s[1,:]], [self.state_size,1])
                a, = self.sess.run(self.network.actions,
                                   feed_dict={self.network.state_ph: s[None]})

                # add temporally-correlated exploration noise to action
                # (using an Ornstein-Uhlenbeck process)
                noise_process = parameters.EXPLO_THETA * \
                    (parameters.EXPLO_MU - noise_process) + \
                    parameters.EXPLO_SIGMA * np.random.randn(self.action_size)
                a += noise_scale * noise_process
                #to respect the bounds:
                a = np.clip(a, self.low_bound, self.high_bound)
                
                s_, v  = self.env.act(a,WH)
                
                #reward  assignation algorithm
                if episode_step==1:
                    r=0
                #elif s[int(self.state_size/2-2)]>(13*TORAD) and s[int(self.state_size/2-2)]<(15*TORAD) and v>0.63 and v<0.67 and a<0:
                #    r=0.1
                else:
                    if v<=0.69:
                        r=0
                        nearlyDone = 0
                    elif v>0.69 and v<=0.75:
                        r=0.00001
                        nearlyDone = 0
                    elif v>0.75 and v<=0.8:
                        r=0.01
                        nearlyDone = 0
                    elif v>0.80:
                        r=0.1
                        if nearlyDone>=3:
                            r=1
                            done = True
                        elif nearlyDone==2:
                            r=0.8
                        elif nearlyDone==1:
                            r=0.25
                        nearlyDone=nearlyDone+1
                    else:
                        r=0
                        nearlyDone = False

                episode_reward += r

                self.buffer.add((s, np.reshape(a, [1,1] ), r, np.reshape(s_, [self.state_size,1]), 0.0 if episode_step<parameters.MAX_EPISODE_STEPS-1 else 1.0)) #, 0.0 if done else 1.0

                # update network weights to fit a minibatch of experience
                if self.total_steps % parameters.TRAINING_FREQ == 0 and \
                        len(self.buffer) >= parameters.BATCH_SIZE:

                    minibatch = self.buffer.sample()

                    _, _,critic_loss = self.sess.run([self.network.critic_train_op, self.network.actor_train_op,self.network.critic_loss],
                                         feed_dict={
                        self.network.state_ph: np.asarray([elem[0] for elem in minibatch]),
                        self.network.action_ph: np.asarray([elem[1] for elem in minibatch]),
                        self.network.reward_ph: np.asarray([elem[2] for elem in minibatch]),
                        self.network.next_state_ph: np.asarray([elem[3] for elem in minibatch]),
                        self.network.is_not_terminal_ph: np.asarray([elem[4] for elem in minibatch])})

                    # update target networks
                    _ = self.sess.run(self.network.update_slow_targets_op)

                s = s_
                episode_step += 1
                self.total_steps += 1
            if ep % parameters.DISP_EP_REWARD_FREQ == 0:
                print('Episode %2i, initial heading: %7.3f, Reward: %7.3f, Final noise scale: %7.3f, critic loss: %7.3f' %
                      (ep, hdg0[0]*(1/TORAD), episode_reward, noise_scale,critic_loss))
            DISPLAYER.add_reward(episode_reward)
            # We save CNN weights every 500 epochs
            if ep % 500 == 0 and ep != 0:
                self.save("NetworkParam/"+ str(ep) +"_epochs")
        self.save("NetworkParam/"+"FinalParam")
예제 #15
0
    def process(self, sess, total_steps, summary_writer, summary_op, score_input):

        start_time = time.time()
        buffer = []
        done = False
        episode_step = 0

        # copy weights from global to local
        sess.run(self.update_network)

        start_lstm_state = self.local_network.lstm_state_out

        for i in range(UPDATE_FREQ):

            pi, value = self.local_network.run_policy_and_value(sess,
                                                                self.state)

            a = np.random.choice(ACTION_SIZE, p=pi)
            s_, r, terminal, _ = self.env.act(a)

            self.episode_reward += r

            # clip reward
            r = np.clip(r, -1, 1)
            buffer.append((self.state, a, r, value))

            episode_step += 1
            self.worker_total_steps += 1
            self.state = s_

            if terminal:
                done = True
                self.worker_total_eps += 1

                DISPLAYER.add_reward(self.episode_reward, self.thread_index)

                if (self.thread_index == 1 and
                        self.worker_total_eps % DISP_REWARD_FREQ == 0):
                    cur_learning_rate = self._anneal_learning_rate(total_steps)
                    print('Episode %i, Reward %i, Steps %i, LR %g' %
                          (self.worker_total_eps, self.episode_reward,
                           episode_step, cur_learning_rate))

                self._record_score(sess, summary_writer, summary_op, score_input,
                                   self.episode_reward, total_steps)

                self.episode_reward = 0
                self.env.reset()
                self.local_network.reset_state()

                render = (DISPLAY and self.thread_index == 1 and
                          (self.worker_total_eps - 1) % RENDER_FREQ == 0)
                self.env.set_render(render)

                break

        batch_s = deque()
        batch_a = deque()
        batch_td = deque()
        batch_R = deque()

        # Bootstrapping
        R = 0.0
        if not done:
            R = self.local_network.run_value(sess, self.state)

        # compute and accumulate gradients
        for i in range(len(buffer) - 1, -1, -1):
            si, ai, ri, Vi = buffer[i]
            R = ri + GAMMA * R
            td = R - Vi
            a = np.zeros([ACTION_SIZE])
            a[ai] = 1

            batch_s.appendleft(si)
            batch_a.appendleft(a)
            batch_td.appendleft(td)
            batch_R.appendleft(R)

        cur_learning_rate = self._anneal_learning_rate(total_steps)

        feed_dict = {self.local_network.state: batch_s,
                     self.local_network.action: batch_a,
                     self.local_network.td_error: batch_td,
                     self.local_network.reward: batch_R,
                     self.local_network.initial_lstm_state: start_lstm_state,
                     self.local_network.step_size: [len(batch_a)],
                     self.learning_rate_input: cur_learning_rate}
        sess.run(self.apply_gradients, feed_dict=feed_dict)

        if done and (self.thread_index == 1) and \
                (self.worker_total_eps % PERF_FREQ == 0 or
                 self.worker_total_eps == 15):
            global_time = time.time() - self.start_time
            steps_per_sec = total_steps / global_time
            print("### Performance : {} STEPS in {:.0f} sec."
                  "{:.0f} STEPS/sec. {:.2f}M STEPS/hour ###".format(
                      total_steps,  global_time, steps_per_sec,
                      steps_per_sec * 3600 / 1000000.))

        elapsed_time = time.time() - start_time
        return elapsed_time, done, episode_step
예제 #16
0
    def run(self):

        self.nb_ep = 1
        self.total_steps = 0

        for self.nb_ep in range(1, parameters.TRAINING_STEPS + 1):

            episode_reward = 0
            episode_step = 0
            done = False
            memory = deque()

            # Initial state
            s = self.env.reset()
            max_steps = parameters.MAX_EPISODE_STEPS + self.nb_ep // parameters.EP_ELONGATION

            while episode_step < max_steps and not done:

                if random.random() < self.epsilon:
                    a = self.env.random()
                else:
                    # choose action based on deterministic policy
                    a, = self.sess.run(self.network.actions,
                                       feed_dict={self.network.state_ph: [s]})

                # Decay epsilon
                if self.epsilon > parameters.EPSILON_STOP:
                    self.epsilon -= parameters.EPSILON_DECAY

                s_, r, done, info = self.env.act(a)
                memory.append((s, a, r, s_, 0.0 if done else 1.0))

                if len(memory) > parameters.N_STEP_RETURN:
                    s_mem, a_mem, r_mem, ss_mem, done_mem = memory.popleft()
                    discount_R = 0
                    for i, (si, ai, ri, s_i, di) in enumerate(memory):
                        discount_R += ri * parameters.DISCOUNT**(i + 1)
                    self.buffer.add(s_mem, a_mem, discount_R, s_, done)

                # update network weights to fit a minibatch of experience
                if self.total_steps % parameters.TRAINING_FREQ == 0 and \
                        len(self.buffer) >= parameters.BATCH_SIZE:

                    minibatch = self.buffer.sample(parameters.BATCH_SIZE,
                                                   self.beta)

                    if self.beta <= parameters.BETA_STOP:
                        self.beta += parameters.BETA_INCR

                    td_errors, _, _ = self.sess.run(
                        [
                            self.network.td_errors,
                            self.network.critic_train_op,
                            self.network.actor_train_op
                        ],
                        feed_dict={
                            self.network.state_ph: minibatch[0],
                            self.network.action_ph: minibatch[1],
                            self.network.reward_ph: minibatch[2],
                            self.network.next_state_ph: minibatch[3],
                            self.network.is_not_terminal_ph: minibatch[4]
                        })

                    self.buffer.update_priorities(minibatch[6],
                                                  td_errors + 1e-6)
                    # update target networks
                    _ = self.sess.run(self.network.update_slow_targets_op)

                episode_reward += r
                s = s_
                episode_step += 1
                self.total_steps += 1

            self.nb_ep += 1

            if self.nb_ep % parameters.DISP_EP_REWARD_FREQ == 0:
                print(
                    'Episode %2i, Reward: %7.3f, Steps: %i, Epsilon : %7.3f, Max steps : %i'
                    % (self.nb_ep, episode_reward, episode_step, self.epsilon,
                       max_steps))

            DISPLAYER.add_reward(episode_reward)

            if episode_reward > self.best_run and self.nb_ep > 100:
                self.best_run = episode_reward
                print("Best agent ! ", episode_reward)
                SAVER.save('best')

            if self.nb_ep % parameters.SAVE_FREQ == 0:
                SAVER.save(self.nb_ep)
예제 #17
0
    def play(self, sess, number_run, path=''):
        print("Playing", self.name, "for", number_run, "runs")

        with sess.as_default(), sess.graph.as_default():
            hdg0_rand_vec = [0, 7, 13]
            '''
            WIND CONDITIONS
            '''
            mean = 45 * TORAD
            std = 0 * TORAD
            wind_samples = 10
            w = wind(mean=mean, std=std, samples=wind_samples)

            try:
                for i in range(number_run):

                    # Reset the local network to the global
                    if self.name != 'global':
                        sess.run(self.update_local_vars)

                    WH = w.generateWind()
                    hdg0_rand = hdg0_rand_vec[i]
                    hdg0 = hdg0_rand * TORAD * np.ones(10)
                    s = self.env.reset(hdg0, WH)
                    episode_reward = 0
                    episode_step = 0
                    v_episode = []
                    i_episode = []
                    done = False

                    #self.lstm_state = self.network.lstm_state_init

                    while (not done and episode_step < 70):
                        i_episode.append(round(s[0][-1] / TORAD))
                        s = np.reshape([s[0, :], s[1, :]],
                                       [2 * self.state_size, 1])
                        # Prediction of the policy
                        feed_dict = {self.network.inputs: [s]}
                        policy, value = sess.run(
                            [self.network.policy, self.network.value],
                            feed_dict=feed_dict)

                        policy = policy[0]

                        # Choose an action according to the policy
                        action = np.random.choice([1.5, 0, -1.5], p=policy)
                        s_, r = self.env.act(action, WH)
                        if episode_step > 12:
                            if np.mean(v_episode[-4:]) > 0.8:
                                #done=True
                                print("Done!")
                            else:
                                done = False
                        episode_reward += r
                        v_episode.append(r)
                        episode_step += 1
                        s = s_
                    DISPLAYER.displayVI(v_episode, i_episode, i)

                    print("Episode reward :", episode_reward)

            except KeyboardInterrupt as e:
                pass

            finally:
                print("End of the demo")