class Learner:

    def __init__(self, gamma, env_name):
        self.env_name = env_name
        self.action_space = gym.make(self.env_name).action_space.n
        self.q_network = QNetwork(self.action_space)
        self.target_q_network = QNetwork(self.action_space)
        self.gamma = gamma
        self.optimizer = tf.keras.optimizers.Adam(lr=0.001)

    def define_network(self):
        env = gym.make(self.env_name)
        state = env.reset()
        self.q_network(np.atleast_2d(state))
        self.target_q_network(np.atleast_2d(state))
        self.target_q_network.set_weights(self.q_network.get_weights())
        current_weights = self.q_network.get_weights()
        return current_weights

    def update_network(self, minibatchs):

        indices_all = []
        td_errors_all = []

        for (indices, weights, transitions) in minibatchs:

            states, actions, rewards, next_states, dones = zip(*transitions)

            states = np.vstack(states)
            actions = np.array(actions)
            rewards = np.vstack(rewards)
            next_states = np.vstack(next_states)
            dones = np.vstack(dones)

            next_qvalues = self.q_network(next_states)
            next_actions = tf.cast(tf.argmax(next_qvalues, axis=1), tf.int32)
            next_actions_onehot = tf.one_hot(next_actions, self.action_space)
            next_maxQ = tf.reduce_sum(
                next_qvalues * next_actions_onehot, axis=1, keepdims=True)
            TQ = rewards + self.gamma * (1 - dones) * next_maxQ

            with tf.GradientTape() as tape:
                qvalues = self.q_network(states)
                actions_onehot = tf.one_hot(actions, self.action_space)
                Q = tf.reduce_sum(qvalues * actions_onehot, axis=1, keepdims=True)
                td_errors = tf.square(TQ - Q)
                loss = tf.reduce_mean(weights * td_errors)

            grads = tape.gradient(loss, self.q_network.trainable_variables)
            grads, _ = tf.clip_by_global_norm(grads, 40.0)
            self.optimizer.apply_gradients(
                zip(grads, self.q_network.trainable_variables))

            indices_all += indices
            td_errors_all += td_errors.numpy().flatten().tolist()

        current_weights = self.q_network.get_weights()
        return current_weights, indices_all, td_errors_all
Exemplo n.º 2
0
class Agent():
    """Interacts with and learns from the environment."""
    def __init__(self, state_shape, action_size, buffer_size, batch_size,
                 gamma, tau, learning_rate, update_every, seed):
        """Initialize an Agent object.

        Params
        ======
            state_size (int): dimension of each state
            action_size (int): dimension of each action
            buffer_size (int): replay buffer size
            batch_size (int): minibatch size
            gamma (float): discount factor
            tau (float): used for soft update of target parameters
            learning_rate (float): learning rate
            update_every (int): how many steps between network updates
            device (torch.Device): pytorch device
            seed (int): random seed
        """
        self.state_shape = state_shape
        self.action_size = action_size
        self.batch_size = batch_size
        self.gamma = gamma
        self.tau = tau
        self.update_every = update_every
        self.seed = random.seed(seed)

        # Q-Network
        self.qnetwork_local = QNetwork(action_size)
        self.qnetwork_target = QNetwork(action_size)

        #self.optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)

        self.optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.00025,
                                                     momentum=0.95)
        self.loss_fn = tf.keras.losses.MeanSquaredError()

        # Replay memory
        self.memory = ReplayBuffer(action_size, buffer_size, batch_size, seed)

        # Initialize time step (for updating every self.update_every steps)
        self.t_step = 0

    def step(self, state, action, reward, next_state, done):
        """Adds new experience to the replay buffer and learns from a subset of memories.

        Params
        ======
        state (array_like): initial state
        action (int): chosen action
        reward (int): reward given
        next_state (array_like): next state
        done (bool): True if the episode is finished
        """

        # Save experience in replay memory
        self.memory.add(state, action, reward, next_state, done)

        # Learn every self.update_every time steps.
        self.t_step = (self.t_step + 1) % self.update_every
        if self.t_step == 0:
            # If enough samples are available in memory, get random subset and learn
            if len(self.memory) > self.batch_size:
                experiences = self.memory.sample()

                self.learn(experiences)

    def act(self, state, epsilon=0.):
        """Returns actions for given state as per current policy.

        Params
        ======
            state (array_like): current state
            epsilon (float): epsilon, for epsilon-greedy action selection
        """
        #state = tf.expand_dims(state, axis=0)
        action_values = self.qnetwork_local(state)

        # Epsilon-greedy action selection
        if random.random() > epsilon:
            return np.argmax(action_values.numpy())
        else:
            return random.choice(np.arange(self.action_size))

    def learn(self, experiences):
        """Update value parameters using given batch of experience tuples.

        Params
        ======
            experiences (Tuple[torch.Variable]): tuple of (s, a, r, s', done) tuples
        """
        states, actions, rewards, next_states, dones = experiences

        # Get max predicted Q values (for next states) from target model
        Q_targets_next = self.qnetwork_target(next_states)
        Q_targets_next = tf.math.reduce_max(Q_targets_next, axis=1)
        Q_targets_next = tf.expand_dims(Q_targets_next, axis=1)

        # Compute Q targets for current states
        Q_targets = rewards + (self.gamma * Q_targets_next * (1 - dones))

        with tf.GradientTape() as tape:
            # Get expected Q values from local model
            Q_expected = self.qnetwork_local(states)
            Q_expected = tf.gather(Q_expected,
                                   indices=actions,
                                   axis=1,
                                   batch_dims=1)
            loss = self.loss_fn(y_true=Q_targets, y_pred=Q_expected)

        gradients = tape.gradient(loss, self.qnetwork_local.trainable_weights)
        self.optimizer.apply_gradients(
            zip(gradients, self.qnetwork_local.trainable_weights))

        # ------------------- update target network ------------------- #
        #self.soft_update()

    def hard_update(self):
        self.qnetwork_target.set_weights(self.qnetwork_local.get_weights())

    def soft_update(self):
        """Soft update model parameters.

        Params
        ======
            local_model (PyTorch model): weights will be copied from
            target_model (PyTorch model): weights will be copied to
        """
        for target_layer, local_layer in zip(self.qnetwork_target.layers,
                                             self.qnetwork_local.layers):
            if target_layer.trainable:
                for i in range(len(target_layer.trainable_weights)):
                    target_layer.trainable_weights[
                        i] = self.tau * local_layer.trainable_weights[i] + (
                            1.0 - self.tau) * target_layer.trainable_weights[i]
Exemplo n.º 3
0
class Agent:
    """Interacts with and learns from the environment."""
    def __init__(self,
                 state_size,
                 action_size,
                 buffer_size,
                 batch_size,
                 gamma,
                 tau,
                 lr,
                 update_every):
        """Initialize an Agent object.

        Args:
            state_size: Integer. Dimension of each state
            action_size: Integer. Dimension of each action
            buffer_size: Integer. Replay buffer size
            batch_size: Integer. Mini-batch size
            gamma: Float. Discount factor
            tau: Float. For soft update of target parameters
            lr: Float. Learning rate
            update_every: Integer. How often to update the network
        """
        # Environment parameters
        self.state_size = state_size
        self.action_size = action_size

        # Q-Learning
        self.gamma = gamma

        # Q-Network
        self.model_local = QNetwork(state_size, action_size)
        self.model_target = QNetwork(state_size, action_size)
        self.optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
        self.loss_fn = tf.keras.losses.MeanSquaredError(name="mse")
        self.tau = tau
        self.update_every = update_every
        self.batch_size = batch_size

        # Replay memory
        self.memory = ReplayBuffer(action_size, buffer_size, batch_size)

        # Initialize time step (for updating every update_every steps)
        self.t_step = 0

    def __str__(self):
        return 'RL_Agent_Class'

    def __repr__(self):
        return 'RL_Agent_Class'

    def step(self, state, action, reward, next_state, done):
        """Save state on buffer and trigger learn according to update_every

        Args:
            state: The previous state of the environment
            action: Integer. Previous action selected by the agent
            reward: Float. Reward value
            next_state: The current state of the environment
            done: Boolean. Whether the episode is complete
        """
        # Save experience in replay memory
        self.memory.add(state, action, reward, next_state, done)

        # Learn every update_every time steps.
        self.t_step = (self.t_step + 1) % self.update_every
        if self.t_step == 0:

            # If enough samples are available in memory, get random subset and learn
            if len(self.memory) > self.batch_size:
                experiences = self.memory.sample()
                self.learn(experiences)

    def act(self, state, eps=0.):
        """Returns actions for given state as per current policy.

        Args:
            state: A array like object or list with states
            eps: Float. Random value for epsilon-greedy action selection

        Returns:
            An action selected by the network or by the epsilon-greedy method
        """
        # Reshape state
        state = np.expand_dims(state, 0)

        # Predict action
        action_values = self.model_local(state)

        # Epsilon-greedy action selection
        if random.random() > eps:
            return np.argmax(action_values)
        else:
            return random.choice(np.arange(self.action_size))

    def learn(self, experiences):
        """Update value parameters using given batch of experience tuples.

        Args:
            experiences: Tuple. Content of tuple (s, a, r, s', done)
        """
        states, actions, rewards, next_states, dones = experiences

        # Create mask to actions
        mask = tf.one_hot(actions.reshape(-1), self.action_size)

        with tf.GradientTape(persistent=True) as tape:

            # Get expected Q values from local model
            q_expected = tf.reduce_sum(self.model_local(states) * mask, axis=1, keepdims=True)

            # Get max predicted Q values (for next states) from target model
            q_targets_next = tf.reduce_max(self.model_target(next_states, training=True), axis=1)

            # Compute Q targets for current states
            q_targets = tf.add(
                rewards, tf.multiply(
                    self.gamma, q_targets_next, tf.subtract(1.0, dones))
            )

            # Compute loss
            loss = self.loss_fn(q_expected, q_targets)

        # Minimize the loss
        gradients = tape.gradient(loss, self.model_local.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, self.model_local.trainable_variables))

        # Update target network
        self.soft_update()

    def soft_update(self):
        """Soft update model parameters.

        The model is update using:
            θ_target = τ * θ_local + (1 - τ) * θ_target

        """

        # Instantiate weight list
        new_weights = []

        # Apply soft update
        for weights in self.model_local.get_weights():
            new_weights.append(self.tau * weights + (1.0 - self.tau) * weights)

        # Set new weights
        self.model_target.set_weights(new_weights)
class DQNAgent:
    def __init__(self,
                 env_name="BreakoutDeterministic-v4",
                 gamma=0.99,
                 batch_size=32,
                 lr=0.00025,
                 update_period=4,
                 target_update_period=10000,
                 n_frames=4):

        self.env_name = env_name

        self.gamma = gamma

        self.batch_size = batch_size

        self.epsilon_scheduler = (
            lambda steps: max(1.0 - 0.9 * steps / 1000000, 0.1))

        self.update_period = update_period

        self.target_update_period = target_update_period

        env = gym.make(self.env_name)

        self.action_space = env.action_space.n

        self.qnet = QNetwork(self.action_space)

        self.target_qnet = QNetwork(self.action_space)

        self.optimizer = Adam(lr=lr, epsilon=0.01 / self.batch_size)

        self.n_frames = n_frames

        self.use_reward_clipping = True

        self.huber_loss = tf.keras.losses.Huber()

    def learn(self, n_episodes, buffer_size=1000000, logdir="log"):

        logdir = Path(__file__).parent / logdir
        if logdir.exists():
            shutil.rmtree(logdir)
        self.summary_writer = tf.summary.create_file_writer(str(logdir))

        self.replay_buffer = ReplayBuffer(max_len=buffer_size)

        steps = 0
        for episode in range(1, n_episodes + 1):
            env = gym.make(self.env_name)

            frame = preprocess_frame(env.reset())
            frames = collections.deque([frame] * self.n_frames,
                                       maxlen=self.n_frames)

            episode_rewards = 0
            episode_steps = 0
            done = False
            lives = 5

            while not done:

                steps, episode_steps = steps + 1, episode_steps + 1

                epsilon = self.epsilon_scheduler(steps)

                state = np.stack(frames, axis=2)[np.newaxis, ...]

                action = self.qnet.sample_action(state, epsilon=epsilon)

                next_frame, reward, done, info = env.step(action)

                episode_rewards += reward

                frames.append(preprocess_frame(next_frame))

                next_state = np.stack(frames, axis=2)[np.newaxis, ...]

                if info["ale.lives"] != lives:
                    lives = info["ale.lives"]
                    transition = (state, action, reward, next_state, True)
                else:
                    transition = (state, action, reward, next_state, done)

                self.replay_buffer.push(transition)

                if len(self.replay_buffer) > 50000:
                    if steps % self.update_period == 0:
                        loss = self.update_network()
                        with self.summary_writer.as_default():
                            tf.summary.scalar("loss", loss, step=steps)
                            tf.summary.scalar("epsilon", epsilon, step=steps)
                            tf.summary.scalar("buffer_size",
                                              len(self.replay_buffer),
                                              step=steps)
                            tf.summary.scalar("train_score",
                                              episode_rewards,
                                              step=steps)
                            tf.summary.scalar("train_steps",
                                              episode_steps,
                                              step=steps)

                    if steps % self.target_update_period == 0:
                        self.target_qnet.set_weights(self.qnet.get_weights())

                if done:
                    break

            print(
                f"Episode: {episode}, score: {episode_rewards}, steps: {episode_steps}"
            )
            if episode % 20 == 0:
                test_scores, test_steps = self.test_play(n_testplay=1)
                with self.summary_writer.as_default():
                    tf.summary.scalar("test_score", test_scores[0], step=steps)
                    tf.summary.scalar("test_step", test_steps[0], step=steps)

            if episode % 1000 == 0:
                self.qnet.save_weights("checkpoints/qnet")

    def update_network(self):

        #: ミニバッチの作成
        (states, actions, rewards, next_states,
         dones) = self.replay_buffer.get_minibatch(self.batch_size)

        if self.use_reward_clipping:
            rewards = np.clip(rewards, -1, 1)

        next_actions, next_qvalues = self.target_qnet.sample_actions(
            next_states)
        next_actions_onehot = tf.one_hot(next_actions, self.action_space)
        max_next_qvalues = tf.reduce_sum(next_qvalues * next_actions_onehot,
                                         axis=1,
                                         keepdims=True)

        target_q = rewards + self.gamma * (1 - dones) * max_next_qvalues

        with tf.GradientTape() as tape:

            qvalues = self.qnet(states)
            actions_onehot = tf.one_hot(actions.flatten().astype(np.int32),
                                        self.action_space)
            q = tf.reduce_sum(qvalues * actions_onehot, axis=1, keepdims=True)
            loss = self.huber_loss(target_q, q)

        grads = tape.gradient(loss, self.qnet.trainable_variables)
        self.optimizer.apply_gradients(
            zip(grads, self.qnet.trainable_variables))

        return loss

    def test_play(self, n_testplay=1, monitor_dir=None, checkpoint_path=None):

        if checkpoint_path:
            env = gym.make(self.env_name)
            frame = preprocess_frame(env.reset())
            frames = collections.deque([frame] * self.n_frames,
                                       maxlen=self.n_frames)

            state = np.stack(frames, axis=2)[np.newaxis, ...]
            self.qnet(state)
            self.qnet.load_weights(checkpoint_path)

        if monitor_dir:
            monitor_dir = Path(monitor_dir)
            if monitor_dir.exists():
                shutil.rmtree(monitor_dir)
            monitor_dir.mkdir()
            env = gym.wrappers.Monitor(gym.make(self.env_name),
                                       monitor_dir,
                                       force=True,
                                       video_callable=(lambda ep: True))
        else:
            env = gym.make(self.env_name)

        scores = []
        steps = []
        for _ in range(n_testplay):

            frame = preprocess_frame(env.reset())
            frames = collections.deque([frame] * self.n_frames,
                                       maxlen=self.n_frames)

            done = False
            episode_steps = 0
            episode_rewards = 0

            while not done:
                state = np.stack(frames, axis=2)[np.newaxis, ...]
                action = self.qnet.sample_action(state, epsilon=0.05)
                next_frame, reward, done, _ = env.step(action)
                frames.append(preprocess_frame(next_frame))

                episode_rewards += reward
                episode_steps += 1
                if episode_steps > 500 and episode_rewards < 3:
                    #: ゲーム開始(action: 0)しないまま停滞するケースへの対処
                    break

            scores.append(episode_rewards)
            steps.append(episode_steps)

        return scores, steps