def train(params): # setup algorithm dqn = DQN(batch_size=params.get("batch_size"), learning_rate=params.get("learning_rate"), target_update_interval=params.get("target_update_interval"), q_func_factory=QRQFunctionFactory( n_quantiles=params.get("n_quantiles")), n_steps=params.get("train_freq"), gamma=params.get("gamma"), n_critics=1, target_reduction_type="min", use_gpu=True) # setup replay buffer buffer = ReplayBuffer(maxlen=params.get("buffer_size"), env=env) # setup explorers explorer = LinearDecayEpsilonGreedy( start_epsilon=1.0, end_epsilon=params.get("exploration_final_eps"), duration=100000) # start training dqn.fit_online( env, buffer, n_steps=params.get("train_steps"), explorer= explorer, # you don't need this with probablistic policy algorithms tensorboard_dir=log_dir, eval_env=eval_env) dqn.save_model(exp_name)
def test_fit_online_cartpole_with_dqn(): env = gym.make('CartPole-v0') eval_env = gym.make('CartPole-v0') algo = DQN() buffer = ReplayBuffer(1000, env) explorer = LinearDecayEpsilonGreedy() algo.fit_online(env, buffer, explorer, n_epochs=1, eval_env=eval_env, logdir='test_data', tensorboard=False)
def test_fit_online_cartpole_with_dqn(): env = gym.make("CartPole-v0") eval_env = gym.make("CartPole-v0") algo = DQN() buffer = ReplayBuffer(1000, env) explorer = LinearDecayEpsilonGreedy() algo.fit_online( env, buffer, explorer, n_steps=100, eval_env=eval_env, logdir="test_data", )
def test_fit_online_atari_with_dqn(): import d4rl_atari env = ChannelFirst(DummyAtari()) eval_env = ChannelFirst(DummyAtari()) algo = DQN(n_frames=4) buffer = ReplayBuffer(1000, env) explorer = LinearDecayEpsilonGreedy() algo.fit_online( env, buffer, explorer, n_steps=100, eval_env=eval_env, logdir="test_data", ) assert algo.impl.observation_shape == (4, 84, 84)
def test_fit_online_atari_with_dqn(): import d4rl_atari env = gym.make("breakout-mixed-v0", stack=False) eval_env = gym.make("breakout-mixed-v0", stack=False) algo = DQN(n_frames=4) buffer = ReplayBuffer(1000, env) explorer = LinearDecayEpsilonGreedy() algo.fit_online( env, buffer, explorer, n_steps=100, eval_env=eval_env, logdir="test_data", tensorboard=False, ) assert algo.impl.observation_shape == (4, 84, 84)
def train(params): # setup algorithm if pretrain: dqn = DQN(batch_size=params.get("batch_size"), learning_rate=params.get("learning_rate"), target_update_interval=params.get("target_update_interval"), q_func_factory=QRQFunctionFactory( n_quantiles=params.get("n_quantiles")), n_steps=params.get("train_freq"), gamma=params.get("gamma"), n_critics=1, target_reduction_type="min", use_gpu=True) # setup replay buffer buffer = ReplayBuffer(maxlen=params.get("buffer_size"), env=env) # setup explorers explorer = LinearDecayEpsilonGreedy( start_epsilon=1.0, end_epsilon=params.get("exploration_final_eps"), duration=100000) # start training dqn.fit_online( env, buffer, n_steps=params.get("train_steps"), explorer= explorer, # you don't need this with probablistic policy algorithms tensorboard_dir=log_dir, eval_env=eval_env) print("Saving Model") dqn.save_model(exp_name) print("convert buffer to dataset") dataset = buffer.to_mdp_dataset() # save MDPDataset dataset.dump('{0}.h5'.format(exp_name)) print("Loading Dataset for Offline Training") dataset = d3rlpy.dataset.MDPDataset.load('{0}.h5'.format(exp_name)) train_episodes, test_episodes = train_test_split(dataset, test_size=0.2) # The dataset can then be used to train a d3rlpy model cql = DiscreteCQL(learning_rate=6.25e-05, encoder_factory='default', q_func_factory='mean', batch_size=32, n_frames=1, n_steps=1, gamma=0.99, n_critics=1, bootstrap=False, share_encoder=False, target_reduction_type='min', target_update_interval=8000, use_gpu=True, scaler=None, augmentation=None, generator=None, impl=None) cql_exp = params.get("model_name") + "_offline_" + params.get( "environment") cql_log = '../../../logs/' + cql_exp cql.fit(dataset.episodes, eval_episodes=test_episodes, n_epochs=1000, scorers={ 'environment': evaluate_on_environment(env, epsilon=0.05), 'td_error': td_error_scorer, 'discounted_advantage': discounted_sum_of_advantage_scorer, 'value_scale': average_value_estimation_scorer, }, tensorboard_dir=cql_log) cql.save_model(cql_exp)
from d3rlpy.algos import DQN from d3rlpy.online.buffers import ReplayBuffer from d3rlpy.online.explorers import LinearDecayEpsilonGreedy env = gym.make('CartPole-v0') eval_env = gym.make('CartPole-v0') # setup algorithm dqn = DQN(batch_size=32, learning_rate=2.5e-4, target_update_interval=100, use_gpu=False) # replay buffer for experience replay buffer = ReplayBuffer(maxlen=100000, env=env) # epilon-greedy explorer explorer = LinearDecayEpsilonGreedy(start_epsilon=1.0, end_epsilon=0.1, duration=10000) # start training dqn.fit_online(env, buffer, explorer, n_epochs=30, eval_env=eval_env, n_steps_per_epoch=1000, n_updates_per_epoch=100)
eval_env = Atari(gym.make('BreakoutNoFrameskip-v4'), is_eval=True) # setup algorithm dqn = DQN(batch_size=32, learning_rate=2.5e-4, optim_factory=RMSpropFactory(), target_update_interval=10000 // 4, q_func_factory='mean', scaler='pixel', n_frames=4, use_gpu=True) # replay buffer for experience replay buffer = ReplayBuffer(maxlen=1000000, env=env) # epilon-greedy explorer explorer = LinearDecayEpsilonGreedy(start_epsilon=1.0, end_epsilon=0.1, duration=1000000) # start training dqn.fit_online(env, buffer, explorer, eval_env=eval_env, eval_epsilon=0.01, n_steps=50000000, n_steps_per_epoch=100000, update_interval=4, update_start_step=50000)
from d3rlpy.algos import DQN from d3rlpy.online.buffers import ReplayBuffer from d3rlpy.online.explorers import LinearDecayEpsilonGreedy env = gym.make('CartPole-v0') eval_env = gym.make('CartPole-v0') # setup algorithm dqn = DQN(batch_size=32, learning_rate=2.5e-4, target_update_interval=100, use_gpu=False) # replay buffer for experience replay buffer = ReplayBuffer(maxlen=100000, env=env) # epilon-greedy explorer explorer = LinearDecayEpsilonGreedy(start_epsilon=1.0, end_epsilon=0.1, duration=10000) # start training dqn.fit_online(env, buffer, explorer, n_steps=30000, eval_env=eval_env, n_steps_per_epoch=1000, update_start_step=1000)