def __init__(self,
sess,
num_actions,
num_atoms=51,
vmax=10.,
gamma=0.99,
update_horizon=1,
min_replay_history=20000,
update_period=4,
target_update_period=8000,
epsilon_fn=intrinsic_dqn_agent.linearly_decaying_epsilon,
epsilon_train=0.01,
epsilon_eval=0.001,
epsilon_decay_period=250000,
replay_scheme='prioritized',
tf_device='/cpu:*',
use_staging=True,
optimizer=tf.train.AdamOptimizer(
learning_rate=0.0000625, epsilon=0.00015),
summary_writer=None,
summary_writing_frequency=500,
clip_reward=False):
"""Initializes the agent and constructs the components of its graph.
Args:
sess: `tf.Session`, for executing ops.
num_actions: int, number of actions the agent can take at any state.
num_atoms: int, the number of buckets of the value function distribution.
vmax: float, the value distribution support is [-vmax, vmax].
gamma: float, discount factor with the usual RL meaning.
update_horizon: int, horizon at which updates are performed, the 'n' in
n-step update.
min_replay_history: int, number of transitions that should be experienced
before the agent begins training its value function.
update_period: int, period between DQN updates.
target_update_period: int, update period for the target network.
epsilon_fn: function expecting 4 parameters:
(decay_period, step, warmup_steps, epsilon). This function should return
the epsilon value used for exploration during training.
epsilon_train: float, the value to which the agent's epsilon is eventually
decayed during training.
epsilon_eval: float, epsilon used when evaluating the agent.
epsilon_decay_period: int, length of the epsilon decay schedule.
replay_scheme: str, 'prioritized' or 'uniform', the sampling scheme of the
replay memory.
tf_device: str, Tensorflow device on which the agent's graph is executed.
use_staging: bool, when True use a staging area to prefetch the next
training batch, speeding training up by about 30%.
optimizer: tf.train.Optimizer, for training the value function.
summary_writer: SummaryWriter object for outputting training statistics.
Summary writing disabled if set to None.
summary_writing_frequency: int, frequency with which summaries will be
written. Lower values will result in slower training.
clip_reward: bool, whether or not clip the mixture of rewards.
"""
self._clip_reward = clip_reward
self.intrinsic_model = intrinsic_rewards.RNDIntrinsicReward(
sess=sess,
tf_device=tf_device,
summary_writer=summary_writer)
super(RNDRainbowAgent, self).__init__(
sess=sess,
num_actions=num_actions,
num_atoms=num_atoms,
vmax=vmax,
gamma=gamma,
update_horizon=update_horizon,
min_replay_history=min_replay_history,
update_period=update_period,
target_update_period=target_update_period,
epsilon_fn=epsilon_fn,
epsilon_train=epsilon_train,
epsilon_eval=epsilon_eval,
epsilon_decay_period=epsilon_decay_period,
replay_scheme=replay_scheme,
tf_device=tf_device,
use_staging=use_staging,
optimizer=optimizer,
summary_writer=summary_writer,
summary_writing_frequency=summary_writing_frequency)
def __init__(self,
sess,
num_actions,
observation_shape=base_dqn_agent.NATURE_DQN_OBSERVATION_SHAPE,
gamma=0.99,
update_horizon=1,
min_replay_history=20000,
update_period=4,
target_update_period=8000,
epsilon_fn=linearly_decaying_epsilon,
epsilon_train=0.01,
epsilon_eval=0.001,
epsilon_decay_period=250000,
tf_device='/cpu:*',
use_staging=True,
max_tf_checkpoints_to_keep=3,
optimizer=tf.train.RMSPropOptimizer(learning_rate=0.00025,
decay=0.95,
momentum=0.0,
epsilon=0.00001,
centered=True),
summary_writer=None,
summary_writing_frequency=500,
clip_reward=False):
"""Initializes the agent and constructs the components of its graph.
Args:
sess: `tf.Session`, for executing ops.
num_actions: int, number of actions the agent can take at any state.
observation_shape: tuple of ints describing the observation shape.
gamma: float, discount factor with the usual RL meaning.
update_horizon: int, horizon at which updates are performed, the 'n' in
n-step update.
min_replay_history: int, number of transitions that should be experienced
before the agent begins training its value function.
update_period: int, period between DQN updates.
target_update_period: int, update period for the target network.
epsilon_fn: function expecting 4 parameters:
(decay_period, step, warmup_steps, epsilon). This function should return
the epsilon value used for exploration during training.
epsilon_train: float, the value to which the agent's epsilon is eventually
decayed during training.
epsilon_eval: float, epsilon used when evaluating the agent.
epsilon_decay_period: int, length of the epsilon decay schedule.
tf_device: str, Tensorflow device on which the agent's graph is executed.
use_staging: bool, when True use a staging area to prefetch the next
training batch, speeding training up by about 30%.
max_tf_checkpoints_to_keep: int, the number of TensorFlow checkpoints to
keep.
optimizer: `tf.train.Optimizer`, for training the value function.
summary_writer: SummaryWriter object for outputting training statistics.
Summary writing disabled if set to None.
summary_writing_frequency: int, frequency with which summaries will be
written. Lower values will result in slower training.
clip_reward: bool, whether or not to clip the reward after adding
the intrinsic reward.
"""
self._clip_reward = clip_reward
self.intrinsic_model = intrinsic_rewards.RNDIntrinsicReward(
sess=sess, tf_device=tf_device, summary_writer=summary_writer)
super(RNDDQNAgent, self).__init__(
sess=sess,
num_actions=num_actions,
observation_shape=observation_shape,
gamma=gamma,
update_horizon=update_horizon,
min_replay_history=min_replay_history,
update_period=update_period,
target_update_period=target_update_period,
epsilon_fn=epsilon_fn,
epsilon_train=epsilon_train,
epsilon_eval=epsilon_eval,
epsilon_decay_period=epsilon_decay_period,
tf_device=tf_device,
use_staging=use_staging,
optimizer=optimizer,
summary_writer=summary_writer,
summary_writing_frequency=summary_writing_frequency)