def setUp(self) -> None:
self.env = gym.make("CartPole-v0")
self.net = Mock(return_value=Tensor([[0.0, 100.0]]))
self.state = [self.env.reset()]
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.value_agent = ValueAgent(self.net, self.env.action_space.n)
def __init__(
self,
env: str,
eps_start: float = 1.0,
eps_end: float = 0.02,
eps_last_frame: int = 150000,
sync_rate: int = 1000,
gamma: float = 0.99,
learning_rate: float = 1e-4,
batch_size: int = 32,
replay_size: int = 100000,
warm_start_size: int = 10000,
avg_reward_len: int = 100,
min_episode_reward: int = -21,
seed: int = 123,
batches_per_epoch: int = 1000,
n_steps: int = 1,
**kwargs,
):
"""
Args:
env: gym environment tag
eps_start: starting value of epsilon for the epsilon-greedy exploration
eps_end: final value of epsilon for the epsilon-greedy exploration
eps_last_frame: the final frame in for the decrease of epsilon. At this frame espilon = eps_end
sync_rate: the number of iterations between syncing up the target network with the train network
gamma: discount factor
learning_rate: learning rate
batch_size: size of minibatch pulled from the DataLoader
replay_size: total capacity of the replay buffer
warm_start_size: how many random steps through the environment to be carried out at the start of
training to fill the buffer with a starting point
avg_reward_len: how many episodes to take into account when calculating the avg reward
min_episode_reward: the minimum score that can be achieved in an episode. Used for filling the avg buffer
before training begins
seed: seed value for all RNG used
batches_per_epoch: number of batches per epoch
n_steps: size of n step look ahead
"""
super().__init__()
# Environment
self.exp = None
self.env = self.make_environment(env, seed)
self.test_env = self.make_environment(env)
self.obs_shape = self.env.observation_space.shape
self.n_actions = self.env.action_space.n
# Model Attributes
self.buffer = None
self.dataset = None
self.net = None
self.target_net = None
self.build_networks()
self.agent = ValueAgent(
self.net,
self.n_actions,
eps_start=eps_start,
eps_end=eps_end,
eps_frames=eps_last_frame,
)
# Hyperparameters
self.sync_rate = sync_rate
self.gamma = gamma
self.lr = learning_rate
self.batch_size = batch_size
self.replay_size = replay_size
self.warm_start_size = warm_start_size
self.batches_per_epoch = batches_per_epoch
self.n_steps = n_steps
self.save_hyperparameters()
# Metrics
self.total_episode_steps = [0]
self.total_rewards = [0]
self.done_episodes = 0
self.total_steps = 0
# Average Rewards
self.avg_reward_len = avg_reward_len
for _ in range(avg_reward_len):
self.total_rewards.append(
torch.tensor(min_episode_reward, device=self.device))
self.avg_rewards = float(
np.mean(self.total_rewards[-self.avg_reward_len:]))
self.state = self.env.reset()
def __init__(
self,
env: str,
eps_start: float = 1.0,
eps_end: float = 0.02,
eps_last_frame: int = 150000,
sync_rate: int = 1000,
gamma: float = 0.99,
learning_rate: float = 1e-4,
batch_size: int = 32,
replay_size: int = 100000,
warm_start_size: int = 10000,
avg_reward_len: int = 100,
min_episode_reward: int = -21,
seed: int = 123,
batches_per_epoch: int = 1000,
n_steps: int = 1,
**kwargs,
):
"""
PyTorch Lightning implementation of `DQN <https://arxiv.org/abs/1312.5602>`_
Paper authors: Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Alex Graves,
Ioannis Antonoglou, Daan Wierstra, Martin Riedmiller.
Model implemented by:
- `Donal Byrne <https://github.com/djbyrne>`
Example:
>>> from pl_bolts.models.rl.dqn_model import DQN
...
>>> model = DQN("PongNoFrameskip-v4")
Train::
trainer = Trainer()
trainer.fit(model)
Args:
env: gym environment tag
eps_start: starting value of epsilon for the epsilon-greedy exploration
eps_end: final value of epsilon for the epsilon-greedy exploration
eps_last_frame: the final frame in for the decrease of epsilon. At this frame espilon = eps_end
sync_rate: the number of iterations between syncing up the target network with the train network
gamma: discount factor
learning_rate: learning rate
batch_size: size of minibatch pulled from the DataLoader
replay_size: total capacity of the replay buffer
warm_start_size: how many random steps through the environment to be carried out at the start of
training to fill the buffer with a starting point
avg_reward_len: how many episodes to take into account when calculating the avg reward
min_episode_reward: the minimum score that can be achieved in an episode. Used for filling the avg buffer
before training begins
seed: seed value for all RNG used
batches_per_epoch: number of batches per epoch
n_steps: size of n step look ahead
Note:
This example is based on:
https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On-Second-Edition/blob/master/Chapter06/02_dqn_pong.py
Note:
Currently only supports CPU and single GPU training with `distributed_backend=dp`
"""
super().__init__()
# Environment
self.exp = None
self.env = self.make_environment(env, seed)
self.test_env = self.make_environment(env)
self.obs_shape = self.env.observation_space.shape
self.n_actions = self.env.action_space.n
# Model Attributes
self.buffer = None
self.dataset = None
self.net = None
self.target_net = None
self.build_networks()
self.agent = ValueAgent(
self.net,
self.n_actions,
eps_start=eps_start,
eps_end=eps_end,
eps_frames=eps_last_frame,
)
# Hyperparameters
self.sync_rate = sync_rate
self.gamma = gamma
self.lr = learning_rate
self.batch_size = batch_size
self.replay_size = replay_size
self.warm_start_size = warm_start_size
self.batches_per_epoch = batches_per_epoch
self.n_steps = n_steps
self.save_hyperparameters()
# Metrics
self.total_episode_steps = [0]
self.total_rewards = [0]
self.done_episodes = 0
self.total_steps = 0
# Average Rewards
self.avg_reward_len = avg_reward_len
for _ in range(avg_reward_len):
self.total_rewards.append(
torch.tensor(min_episode_reward, device=self.device)
)
self.avg_rewards = float(
np.mean(self.total_rewards[-self.avg_reward_len:])
)
self.state = self.env.reset()
def __init__(
self,
env: str,
gpus: int = 0,
eps_start: float = 1.0,
eps_end: float = 0.02,
eps_last_frame: int = 150000,
sync_rate: int = 1000,
gamma: float = 0.99,
learning_rate: float = 1e-4,
batch_size: int = 32,
replay_size: int = 100000,
warm_start_size: int = 10000,
num_samples: int = 500,
**kwargs,
):
"""
PyTorch Lightning implementation of `DQN <https://arxiv.org/abs/1312.5602>`_
Paper authors: Volodymyr Mnih, Koray Kavukcuoglu, David Silver, Alex Graves,
Ioannis Antonoglou, Daan Wierstra, Martin Riedmiller.
Model implemented by:
- `Donal Byrne <https://github.com/djbyrne>`
Example:
>>> from pl_bolts.models.rl.dqn_model import DQN
...
>>> model = DQN("PongNoFrameskip-v4")
Train::
trainer = Trainer()
trainer.fit(model)
Args:
env: gym environment tag
gpus: number of gpus being used
eps_start: starting value of epsilon for the epsilon-greedy exploration
eps_end: final value of epsilon for the epsilon-greedy exploration
eps_last_frame: the final frame in for the decrease of epsilon. At this frame espilon = eps_end
sync_rate: the number of iterations between syncing up the target network with the train network
gamma: discount factor
learning_rate: learning rate
batch_size: size of minibatch pulled from the DataLoader
replay_size: total capacity of the replay buffer
warm_start_size: how many random steps through the environment to be carried out at the start of
training to fill the buffer with a starting point
num_samples: the number of samples to pull from the dataset iterator and feed to the DataLoader
.. note::
This example is based on:
https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On-Second-Edition\
/blob/master/Chapter06/02_dqn_pong.py
.. note:: Currently only supports CPU and single GPU training with `distributed_backend=dp`
"""
super().__init__()
# Environment
self.env = wrappers.make_env(env)
self.env.seed(123)
self.obs_shape = self.env.observation_space.shape
self.n_actions = self.env.action_space.n
# Model Attributes
self.buffer = None
self.source = None
self.dataset = None
self.net = None
self.target_net = None
self.build_networks()
self.agent = ValueAgent(
self.net,
self.n_actions,
eps_start=eps_start,
eps_end=eps_end,
eps_frames=eps_last_frame,
)
# Hyperparameters
self.sync_rate = sync_rate
self.gamma = gamma
self.lr = learning_rate
self.batch_size = batch_size
self.replay_size = replay_size
self.warm_start_size = warm_start_size
self.sample_len = num_samples
self.save_hyperparameters()
# Metrics
self.total_reward = 0
self.episode_reward = 0
self.episode_count = 0
self.episode_steps = 0
self.total_episode_steps = 0
self.reward_list = []
for _ in range(100):
self.reward_list.append(-21)
self.avg_reward = -21
def setUp(self) -> None:
self.env = gym.make("CartPole-v0")
self.net = Mock(return_value=torch.Tensor([[0.0, 100.0]]))
self.state = torch.tensor(self.env.reset())
self.device = self.state.device
self.value_agent = ValueAgent(self.net, self.env.action_space.n)