def setUp(self) -> None: self.env = gym.make("CartPole-v0") self.net = Mock(return_value=Tensor([[0.0, 100.0]])) self.state = [self.env.reset()] self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.value_agent = ValueAgent(self.net, self.env.action_space.n)
def __init__( self, env: str, eps_start: float = 1.0, eps_end: float = 0.02, eps_last_frame: int = 150000, sync_rate: int = 1000, gamma: float = 0.99, learning_rate: float = 1e-4, batch_size: int = 32, replay_size: int = 100000, warm_start_size: int = 10000, avg_reward_len: int = 100, min_episode_reward: int = -21, seed: int = 123, batches_per_epoch: int = 1000, n_steps: int = 1, **kwargs, ): """ Args: env: gym environment tag eps_start: starting value of epsilon for the epsilon-greedy exploration eps_end: final value of epsilon for the epsilon-greedy exploration eps_last_frame: the final frame in for the decrease of epsilon. At this frame espilon = eps_end sync_rate: the number of iterations between syncing up the target network with the train network gamma: discount factor learning_rate: learning rate batch_size: size of minibatch pulled from the DataLoader replay_size: total capacity of the replay buffer warm_start_size: how many random steps through the environment to be carried out at the start of training to fill the buffer with a starting point avg_reward_len: how many episodes to take into account when calculating the avg reward min_episode_reward: the minimum score that can be achieved in an episode. Used for filling the avg buffer before training begins seed: seed value for all RNG used batches_per_epoch: number of batches per epoch n_steps: size of n step look ahead """ super().__init__() # Environment self.exp = None self.env = self.make_environment(env, seed) self.test_env = self.make_environment(env) self.obs_shape = self.env.observation_space.shape self.n_actions = self.env.action_space.n # Model Attributes self.buffer = None self.dataset = None self.net = None self.target_net = None self.build_networks() self.agent = ValueAgent( self.net, self.n_actions, eps_start=eps_start, eps_end=eps_end, eps_frames=eps_last_frame, ) # Hyperparameters self.sync_rate = sync_rate self.gamma = gamma self.lr = learning_rate self.batch_size = batch_size self.replay_size = replay_size self.warm_start_size = warm_start_size self.batches_per_epoch = batches_per_epoch self.n_steps = n_steps self.save_hyperparameters() # Metrics self.total_episode_steps = [0] self.total_rewards = [0] self.done_episodes = 0 self.total_steps = 0 # Average Rewards self.avg_reward_len = avg_reward_len for _ in range(avg_reward_len): self.total_rewards.append( torch.tensor(min_episode_reward, device=self.device)) self.avg_rewards = float( np.mean(self.total_rewards[-self.avg_reward_len:])) self.state = self.env.reset()
def __init__( self, env: str, eps_start: float = 1.0, eps_end: float = 0.02, eps_last_frame: int = 150000, sync_rate: int = 1000, gamma: float = 0.99, learning_rate: float = 1e-4, batch_size: int = 32, replay_size: int = 100000, warm_start_size: int = 10000, avg_reward_len: int = 100, min_episode_reward: int = -21, seed: int = 123, batches_per_epoch: int = 1000, n_steps: int = 1, **kwargs, ): """ PyTorch Lightning implementation of `DQN <https://arxiv.org/abs/1312.5602>`_ Paper authors: Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Alex Graves, Ioannis Antonoglou, Daan Wierstra, Martin Riedmiller. Model implemented by: - `Donal Byrne <https://github.com/djbyrne>` Example: >>> from pl_bolts.models.rl.dqn_model import DQN ... >>> model = DQN("PongNoFrameskip-v4") Train:: trainer = Trainer() trainer.fit(model) Args: env: gym environment tag eps_start: starting value of epsilon for the epsilon-greedy exploration eps_end: final value of epsilon for the epsilon-greedy exploration eps_last_frame: the final frame in for the decrease of epsilon. At this frame espilon = eps_end sync_rate: the number of iterations between syncing up the target network with the train network gamma: discount factor learning_rate: learning rate batch_size: size of minibatch pulled from the DataLoader replay_size: total capacity of the replay buffer warm_start_size: how many random steps through the environment to be carried out at the start of training to fill the buffer with a starting point avg_reward_len: how many episodes to take into account when calculating the avg reward min_episode_reward: the minimum score that can be achieved in an episode. Used for filling the avg buffer before training begins seed: seed value for all RNG used batches_per_epoch: number of batches per epoch n_steps: size of n step look ahead Note: This example is based on: https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On-Second-Edition/blob/master/Chapter06/02_dqn_pong.py Note: Currently only supports CPU and single GPU training with `distributed_backend=dp` """ super().__init__() # Environment self.exp = None self.env = self.make_environment(env, seed) self.test_env = self.make_environment(env) self.obs_shape = self.env.observation_space.shape self.n_actions = self.env.action_space.n # Model Attributes self.buffer = None self.dataset = None self.net = None self.target_net = None self.build_networks() self.agent = ValueAgent( self.net, self.n_actions, eps_start=eps_start, eps_end=eps_end, eps_frames=eps_last_frame, ) # Hyperparameters self.sync_rate = sync_rate self.gamma = gamma self.lr = learning_rate self.batch_size = batch_size self.replay_size = replay_size self.warm_start_size = warm_start_size self.batches_per_epoch = batches_per_epoch self.n_steps = n_steps self.save_hyperparameters() # Metrics self.total_episode_steps = [0] self.total_rewards = [0] self.done_episodes = 0 self.total_steps = 0 # Average Rewards self.avg_reward_len = avg_reward_len for _ in range(avg_reward_len): self.total_rewards.append( torch.tensor(min_episode_reward, device=self.device) ) self.avg_rewards = float( np.mean(self.total_rewards[-self.avg_reward_len:]) ) self.state = self.env.reset()
def __init__( self, env: str, gpus: int = 0, eps_start: float = 1.0, eps_end: float = 0.02, eps_last_frame: int = 150000, sync_rate: int = 1000, gamma: float = 0.99, learning_rate: float = 1e-4, batch_size: int = 32, replay_size: int = 100000, warm_start_size: int = 10000, num_samples: int = 500, **kwargs, ): """ PyTorch Lightning implementation of `DQN <https://arxiv.org/abs/1312.5602>`_ Paper authors: Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Alex Graves, Ioannis Antonoglou, Daan Wierstra, Martin Riedmiller. Model implemented by: - `Donal Byrne <https://github.com/djbyrne>` Example: >>> from pl_bolts.models.rl.dqn_model import DQN ... >>> model = DQN("PongNoFrameskip-v4") Train:: trainer = Trainer() trainer.fit(model) Args: env: gym environment tag gpus: number of gpus being used eps_start: starting value of epsilon for the epsilon-greedy exploration eps_end: final value of epsilon for the epsilon-greedy exploration eps_last_frame: the final frame in for the decrease of epsilon. At this frame espilon = eps_end sync_rate: the number of iterations between syncing up the target network with the train network gamma: discount factor learning_rate: learning rate batch_size: size of minibatch pulled from the DataLoader replay_size: total capacity of the replay buffer warm_start_size: how many random steps through the environment to be carried out at the start of training to fill the buffer with a starting point num_samples: the number of samples to pull from the dataset iterator and feed to the DataLoader .. note:: This example is based on: https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On-Second-Edition\ /blob/master/Chapter06/02_dqn_pong.py .. note:: Currently only supports CPU and single GPU training with `distributed_backend=dp` """ super().__init__() # Environment self.env = wrappers.make_env(env) self.env.seed(123) self.obs_shape = self.env.observation_space.shape self.n_actions = self.env.action_space.n # Model Attributes self.buffer = None self.source = None self.dataset = None self.net = None self.target_net = None self.build_networks() self.agent = ValueAgent( self.net, self.n_actions, eps_start=eps_start, eps_end=eps_end, eps_frames=eps_last_frame, ) # Hyperparameters self.sync_rate = sync_rate self.gamma = gamma self.lr = learning_rate self.batch_size = batch_size self.replay_size = replay_size self.warm_start_size = warm_start_size self.sample_len = num_samples self.save_hyperparameters() # Metrics self.total_reward = 0 self.episode_reward = 0 self.episode_count = 0 self.episode_steps = 0 self.total_episode_steps = 0 self.reward_list = [] for _ in range(100): self.reward_list.append(-21) self.avg_reward = -21
def setUp(self) -> None: self.env = gym.make("CartPole-v0") self.net = Mock(return_value=torch.Tensor([[0.0, 100.0]])) self.state = torch.tensor(self.env.reset()) self.device = self.state.device self.value_agent = ValueAgent(self.net, self.env.action_space.n)