def run_task(snapshot_config, *_):
"""Run task."""
with LocalRunner(snapshot_config=snapshot_config) as runner:
n_epochs = 100
n_epoch_cycles = 20
sampler_batch_size = 500
num_timesteps = n_epochs * n_epoch_cycles * sampler_batch_size
env = gym.make('PongNoFrameskip-v4')
env = Noop(env, noop_max=30)
env = MaxAndSkip(env, skip=4)
env = EpisodicLife(env)
if 'FIRE' in env.unwrapped.get_action_meanings():
env = FireReset(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = ClipReward(env)
env = StackFrames(env, 4)
env = TfEnv(env)
replay_buffer = SimpleReplayBuffer(
env_spec=env.spec, size_in_transitions=int(5e4), time_horizon=1)
qf = DiscreteCNNQFunction(
env_spec=env.spec,
filter_dims=(8, 4, 3),
num_filters=(32, 64, 64),
strides=(4, 2, 1),
dueling=False)
policy = DiscreteQfDerivedPolicy(env_spec=env.spec, qf=qf)
epilson_greedy_strategy = EpsilonGreedyStrategy(
env_spec=env.spec,
total_timesteps=num_timesteps,
max_epsilon=1.0,
min_epsilon=0.02,
decay_ratio=0.1)
algo = DQN(
env_spec=env.spec,
policy=policy,
qf=qf,
exploration_strategy=epilson_greedy_strategy,
replay_buffer=replay_buffer,
qf_lr=1e-4,
discount=0.99,
min_buffer_size=int(1e4),
double_q=False,
n_train_steps=500,
n_epoch_cycles=n_epoch_cycles,
target_network_update_freq=2,
buffer_batch_size=32)
runner.setup(algo, env)
runner.train(
n_epochs=n_epochs,
n_epoch_cycles=n_epoch_cycles,
batch_size=sampler_batch_size)
def setup_stacked_lunar_lander_env(env_name, num_frames, normalize=False):
from deepmdp.garage_mod.env_wrappers.stack_frames import StackFrames
env = gym.make(env_name)
env = ObfuscateVelocityInformation(env, no_obf=True)
if normalize:
env = MinMaxNormLunar(env)
env = StackFrames(env, num_frames)
return GarageEnv(env)
def setup_lunar_lander_with_image_obs(env_name, num_frames, do_noops=False):
from deepmdp.garage_mod.env_wrappers.stack_frames import StackFrames
env = gym.make(env_name)
env = LunarLanderToImageObservations(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = StackFrames(env, num_frames, do_noops=do_noops)
return GarageEnv(env)
def setup_lunar_lander_with_obfuscated_states(env_name,
num_frames,
do_noops=False):
from deepmdp.garage_mod.env_wrappers.stack_frames import StackFrames
env = gym.make(env_name)
env = ObfuscateVelocityInformation(env)
env = StackFrames(env, num_frames, do_noops=do_noops)
return GarageEnv(env)
def setup_atari_env(env_name, num_frames):
env = gym.make(env_name)
env = Noop(env, noop_max=30)
env = MaxAndSkip(env, skip=4)
env = EpisodicLife(env)
# Fire on reset as some envs are fixed until firing
if 'FIRE' in env.unwrapped.get_action_meanings():
env = FireReset(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = ClipReward(env)
env = StackFrames(env, num_frames)
return GarageEnv(env)
def test_get_time_limit_finds_time_limit():
env = gym.make('PongNoFrameskip-v4')
time_limit = env._max_episode_steps
env = Noop(env, noop_max=30)
env = MaxAndSkip(env, skip=4)
env = EpisodicLife(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = ClipReward(env)
env = StackFrames(env, 4)
env = GymEnv(env)
assert env._max_episode_length == time_limit
def watch_atari(saved_dir, env=None, num_episodes=10):
"""Watch a trained agent play an atari game.
Args:
saved_dir (str): Directory containing the pickle file.
env (str): Environment to run episodes on. If None, the pickled
environment is used.
num_episodes (int): Number of episodes to play. Note that when using
the EpisodicLife wrapper, an episode is considered done when a
life is lost. Defaults to 10.
"""
snapshotter = Snapshotter()
data = snapshotter.load(saved_dir)
if env is not None:
env = gym.make(env)
env = Noop(env, noop_max=30)
env = MaxAndSkip(env, skip=4)
env = EpisodicLife(env)
if 'FIRE' in env.unwrapped.get_action_meanings():
env = FireReset(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = ClipReward(env)
env = StackFrames(env, 4, axis=0)
env = GymEnv(env)
else:
env = data['env']
exploration_policy = data['algo'].exploration_policy
exploration_policy.policy._qf.to('cpu')
ep_rewards = np.asarray([])
for _ in range(num_episodes):
episode_data = rollout(env,
exploration_policy.policy,
animated=True,
pause_per_frame=0.02)
ep_rewards = np.append(ep_rewards, np.sum(episode_data['rewards']))
print('Average Reward {}'.format(np.mean(ep_rewards)))
def dqn_pong(ctxt=None, seed=1, buffer_size=int(5e4), max_episode_length=500):
"""Train DQN on PongNoFrameskip-v4 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by Trainer to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
buffer_size (int): Number of timesteps to store in replay buffer.
max_episode_length (int): Maximum length of an episode, after which an
episode is considered complete. This is used during testing to
minimize the memory required to store a single episode.
"""
set_seed(seed)
with TFTrainer(ctxt) as trainer:
n_epochs = 100
steps_per_epoch = 20
sampler_batch_size = 500
num_timesteps = n_epochs * steps_per_epoch * sampler_batch_size
env = gym.make('PongNoFrameskip-v4')
env = env.unwrapped
env = Noop(env, noop_max=30)
env = MaxAndSkip(env, skip=4)
env = EpisodicLife(env)
if 'FIRE' in env.unwrapped.get_action_meanings():
env = FireReset(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = ClipReward(env)
env = StackFrames(env, 4)
env = GymEnv(env, is_image=True, max_episode_length=max_episode_length)
replay_buffer = PathBuffer(capacity_in_transitions=buffer_size)
qf = DiscreteCNNQFunction(env_spec=env.spec,
filters=(
(32, (8, 8)),
(64, (4, 4)),
(64, (3, 3)),
),
strides=(4, 2, 1),
dueling=False) # yapf: disable
policy = DiscreteQFArgmaxPolicy(env_spec=env.spec, qf=qf)
exploration_policy = EpsilonGreedyPolicy(env_spec=env.spec,
policy=policy,
total_timesteps=num_timesteps,
max_epsilon=1.0,
min_epsilon=0.02,
decay_ratio=0.1)
algo = DQN(env_spec=env.spec,
policy=policy,
qf=qf,
exploration_policy=exploration_policy,
replay_buffer=replay_buffer,
qf_lr=1e-4,
discount=0.99,
min_buffer_size=int(1e4),
double_q=False,
n_train_steps=500,
steps_per_epoch=steps_per_epoch,
target_network_update_freq=2,
buffer_batch_size=32)
trainer.setup(algo, env)
trainer.train(n_epochs=n_epochs, batch_size=sampler_batch_size)
def dqn_atari(ctxt=None,
env=None,
seed=24,
n_workers=psutil.cpu_count(logical=False),
max_episode_length=None,
**kwargs):
"""Train DQN with PongNoFrameskip-v4 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by Trainer to create the snapshotter.
env (str): Name of the atari environment, eg. 'PongNoFrameskip-v4'.
seed (int): Used to seed the random number generator to produce
determinism.
n_workers (int): Number of workers to use. Defaults to the number of
CPU cores available.
max_episode_length (int): Max length of an episode. If None, defaults
to the timelimit specific to the environment. Used by integration
tests.
kwargs (dict): hyperparameters to be saved to variant.json.
"""
assert n_workers > 0
assert env is not None
env = gym.make(env)
env = Noop(env, noop_max=30)
env = MaxAndSkip(env, skip=4)
env = EpisodicLife(env)
if 'FIRE' in env.unwrapped.get_action_meanings():
env = FireReset(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = ClipReward(env)
env = StackFrames(env, 4, axis=0)
env = GymEnv(env, max_episode_length=max_episode_length, is_image=True)
set_seed(seed)
trainer = Trainer(ctxt)
n_epochs = hyperparams['n_epochs']
steps_per_epoch = hyperparams['steps_per_epoch']
sampler_batch_size = hyperparams['sampler_batch_size']
num_timesteps = n_epochs * steps_per_epoch * sampler_batch_size
replay_buffer = PathBuffer(
capacity_in_transitions=hyperparams['buffer_size'])
qf = DiscreteCNNQFunction(
env_spec=env.spec,
image_format='NCHW',
hidden_channels=hyperparams['hidden_channels'],
kernel_sizes=hyperparams['kernel_sizes'],
strides=hyperparams['strides'],
hidden_w_init=(
lambda x: torch.nn.init.orthogonal_(x, gain=np.sqrt(2))),
hidden_sizes=hyperparams['hidden_sizes'])
policy = DiscreteQFArgmaxPolicy(env_spec=env.spec, qf=qf)
exploration_policy = EpsilonGreedyPolicy(
env_spec=env.spec,
policy=policy,
total_timesteps=num_timesteps,
max_epsilon=hyperparams['max_epsilon'],
min_epsilon=hyperparams['min_epsilon'],
decay_ratio=hyperparams['decay_ratio'])
sampler = LocalSampler(agents=exploration_policy,
envs=env,
max_episode_length=env.spec.max_episode_length,
worker_class=FragmentWorker,
n_workers=n_workers)
algo = DQN(env_spec=env.spec,
policy=policy,
qf=qf,
exploration_policy=exploration_policy,
replay_buffer=replay_buffer,
sampler=sampler,
steps_per_epoch=steps_per_epoch,
qf_lr=hyperparams['lr'],
clip_gradient=hyperparams['clip_gradient'],
discount=hyperparams['discount'],
min_buffer_size=hyperparams['min_buffer_size'],
n_train_steps=hyperparams['n_train_steps'],
target_update_freq=hyperparams['target_update_freq'],
buffer_batch_size=hyperparams['buffer_batch_size'])
set_gpu_mode(False)
torch.set_num_threads(1)
if torch.cuda.is_available():
set_gpu_mode(True)
algo.to()
trainer.setup(algo, env)
trainer.train(n_epochs=n_epochs, batch_size=sampler_batch_size)
env.close()
def run_task(snapshot_config, variant_data, *_):
"""Run task.
Args:
snapshot_config (garage.experiment.SnapshotConfig): The snapshot
configuration used by LocalRunner to create the snapshotter.
variant_data (dict): Custom arguments for the task.
*_ (object): Ignored by this function.
"""
with LocalTFRunner(snapshot_config=snapshot_config) as runner:
n_epochs = 100
steps_per_epoch = 20
sampler_batch_size = 500
num_timesteps = n_epochs * steps_per_epoch * sampler_batch_size
env = gym.make('PongNoFrameskip-v4')
env = Noop(env, noop_max=30)
env = MaxAndSkip(env, skip=4)
env = EpisodicLife(env)
if 'FIRE' in env.unwrapped.get_action_meanings():
env = FireReset(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = ClipReward(env)
env = StackFrames(env, 4)
env = TfEnv(env)
replay_buffer = SimpleReplayBuffer(
env_spec=env.spec,
size_in_transitions=variant_data['buffer_size'],
time_horizon=1)
qf = DiscreteCNNQFunction(env_spec=env.spec,
filter_dims=(8, 4, 3),
num_filters=(32, 64, 64),
strides=(4, 2, 1),
dueling=False)
policy = DiscreteQfDerivedPolicy(env_spec=env.spec, qf=qf)
epilson_greedy_strategy = EpsilonGreedyStrategy(
env_spec=env.spec,
total_timesteps=num_timesteps,
max_epsilon=1.0,
min_epsilon=0.02,
decay_ratio=0.1)
algo = DQN(env_spec=env.spec,
policy=policy,
qf=qf,
exploration_strategy=epilson_greedy_strategy,
replay_buffer=replay_buffer,
qf_lr=1e-4,
discount=0.99,
min_buffer_size=int(1e4),
double_q=False,
n_train_steps=500,
steps_per_epoch=steps_per_epoch,
target_network_update_freq=2,
buffer_batch_size=32)
runner.setup(algo, env)
runner.train(n_epochs=n_epochs, batch_size=sampler_batch_size)
def dqn_pong(ctxt=None, seed=1, buffer_size=int(5e4)):
"""Train DQN on PongNoFrameskip-v4 environment.
Args:
ctxt (garage.experiment.ExperimentContext): The experiment
configuration used by LocalRunner to create the snapshotter.
seed (int): Used to seed the random number generator to produce
determinism.
buffer_size (int): Number of timesteps to store in replay buffer.
"""
set_seed(seed)
with LocalTFRunner(ctxt) as runner:
n_epochs = 100
steps_per_epoch = 20
sampler_batch_size = 500
num_timesteps = n_epochs * steps_per_epoch * sampler_batch_size
env = gym.make('PongNoFrameskip-v4')
env = Noop(env, noop_max=30)
env = MaxAndSkip(env, skip=4)
env = EpisodicLife(env)
if 'FIRE' in env.unwrapped.get_action_meanings():
env = FireReset(env)
env = Grayscale(env)
env = Resize(env, 84, 84)
env = ClipReward(env)
env = StackFrames(env, 4)
env = TfEnv(env, is_image=True)
replay_buffer = SimpleReplayBuffer(env_spec=env.spec,
size_in_transitions=buffer_size,
time_horizon=1)
qf = DiscreteCNNQFunction(env_spec=env.spec,
filter_dims=(8, 4, 3),
num_filters=(32, 64, 64),
strides=(4, 2, 1),
dueling=False)
policy = DiscreteQfDerivedPolicy(env_spec=env.spec, qf=qf)
exploration_policy = EpsilonGreedyPolicy(env_spec=env.spec,
policy=policy,
total_timesteps=num_timesteps,
max_epsilon=1.0,
min_epsilon=0.02,
decay_ratio=0.1)
algo = DQN(env_spec=env.spec,
policy=policy,
qf=qf,
exploration_policy=exploration_policy,
replay_buffer=replay_buffer,
qf_lr=1e-4,
discount=0.99,
min_buffer_size=int(1e4),
double_q=False,
n_train_steps=500,
steps_per_epoch=steps_per_epoch,
target_network_update_freq=2,
buffer_batch_size=32)
runner.setup(algo, env)
runner.train(n_epochs=n_epochs, batch_size=sampler_batch_size)