def setUp(self):
self.replay_buffer = ReplayBuffer(buffer_size=2,
batch_size=1,
obs_dim=1,
ac_dim=1)
def __init__(self,
sess,
ob_space,
ac_space,
co_space,
buffer_size,
batch_size,
actor_lr,
critic_lr,
verbose,
tau,
gamma,
layer_norm,
layers,
act_fun,
use_huber,
noise,
target_policy_noise,
target_noise_clip,
scope=None,
zero_fingerprint=False,
fingerprint_dim=2):
"""Instantiate the feed-forward neural network policy.
Parameters
----------
sess : tf.compat.v1.Session
the current TensorFlow session
ob_space : gym.spaces.*
the observation space of the environment
ac_space : gym.spaces.*
the action space of the environment
co_space : gym.spaces.*
the context space of the environment
buffer_size : int
the max number of transitions to store
batch_size : int
SGD batch size
actor_lr : float
actor learning rate
critic_lr : float
critic learning rate
verbose : int
the verbosity level: 0 none, 1 training information, 2 tensorflow
debug
tau : float
target update rate
gamma : float
discount factor
layer_norm : bool
enable layer normalisation
layers : list of int or None
the size of the Neural network for the policy
act_fun : tf.nn.*
the activation function to use in the neural network
use_huber : bool
specifies whether to use the huber distance function as the loss
for the critic. If set to False, the mean-squared error metric is
used instead
noise : float
scaling term to the range of the action space, that is subsequently
used as the standard deviation of Gaussian noise added to the
action if `apply_noise` is set to True in `get_action`
target_policy_noise : float
standard deviation term to the noise from the output of the target
actor policy. See TD3 paper for more.
target_noise_clip : float
clipping term for the noise injected in the target actor policy
scope : str
an upper-level scope term. Used by policies that call this one.
zero_fingerprint : bool
whether to zero the last two elements of the observations for the
actor and critic computations. Used for the worker policy when
fingerprints are being implemented.
fingerprint_dim : int
the number of fingerprint elements in the observation. Used when
trying to zero the fingerprint elements.
Raises
------
AssertionError
if the layers is not a list of at least size 1
"""
super(FeedForwardPolicy, self).__init__(sess=sess,
ob_space=ob_space,
ac_space=ac_space,
co_space=co_space,
buffer_size=buffer_size,
batch_size=batch_size,
actor_lr=actor_lr,
critic_lr=critic_lr,
verbose=verbose,
tau=tau,
gamma=gamma,
layer_norm=layer_norm,
layers=layers,
act_fun=act_fun,
use_huber=use_huber)
# action magnitudes
ac_mag = 0.5 * (ac_space.high - ac_space.low)
self.noise = noise * ac_mag
self.target_policy_noise = np.array([ac_mag * target_policy_noise])
self.target_noise_clip = np.array([ac_mag * target_noise_clip])
self.zero_fingerprint = zero_fingerprint
self.fingerprint_dim = fingerprint_dim
assert len(self.layers) >= 1, \
"Error: must have at least one hidden layer for the policy."
# Compute the shape of the input observation space, which may include
# the contextual term.
ob_dim = self._get_ob_dim(ob_space, co_space)
# =================================================================== #
# Step 1: Create a replay buffer object. #
# =================================================================== #
self.replay_buffer = ReplayBuffer(
buffer_size=self.buffer_size,
batch_size=self.batch_size,
obs_dim=ob_dim[0],
ac_dim=self.ac_space.shape[0],
)
# =================================================================== #
# Step 2: Create input variables. #
# =================================================================== #
with tf.compat.v1.variable_scope("input", reuse=False):
self.terminals1 = tf.compat.v1.placeholder(tf.float32,
shape=(None, 1),
name='terminals1')
self.rew_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, 1),
name='rewards')
self.action_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) +
ac_space.shape,
name='actions')
self.obs_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + ob_dim,
name='obs0')
self.obs1_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + ob_dim,
name='obs1')
# logging of rewards to tensorboard
with tf.compat.v1.variable_scope("input_info", reuse=False):
tf.compat.v1.summary.scalar('rewards', tf.reduce_mean(self.rew_ph))
# =================================================================== #
# Step 3: Create actor and critic variables. #
# =================================================================== #
# Create networks and core TF parts that are shared across setup parts.
with tf.compat.v1.variable_scope("model", reuse=False):
self.actor_tf = self.make_actor(self.obs_ph)
self.critic_tf = [
self.make_critic(self.obs_ph,
self.action_ph,
scope="qf_{}".format(i)) for i in range(2)
]
self.critic_with_actor_tf = [
self.make_critic(self.obs_ph,
self.actor_tf,
reuse=True,
scope="qf_{}".format(i)) for i in range(2)
]
with tf.compat.v1.variable_scope("target", reuse=False):
# create the target actor policy
actor_target = self.make_actor(self.obs1_ph)
# smooth target policy by adding clipped noise to target actions
target_noise = tf.random.normal(tf.shape(actor_target),
stddev=self.target_policy_noise)
target_noise = tf.clip_by_value(target_noise,
-self.target_noise_clip,
self.target_noise_clip)
# clip the noisy action to remain in the bounds
noisy_actor_target = tf.clip_by_value(actor_target + target_noise,
self.ac_space.low,
self.ac_space.high)
# create the target critic policies
critic_target = [
self.make_critic(self.obs1_ph,
noisy_actor_target,
scope="qf_{}".format(i)) for i in range(2)
]
# Create the target update operations.
init, soft = self._setup_target_updates('model', 'target', scope, tau,
verbose)
self.target_init_updates = init
self.target_soft_updates = soft
# =================================================================== #
# Step 4: Setup the optimizers for the actor and critic. #
# =================================================================== #
with tf.compat.v1.variable_scope("Optimizer", reuse=False):
self._setup_actor_optimizer(scope)
self._setup_critic_optimizer(critic_target, scope)
tf.compat.v1.summary.scalar('actor_loss', self.actor_loss)
tf.compat.v1.summary.scalar('Q1_loss', self.critic_loss[0])
tf.compat.v1.summary.scalar('Q2_loss', self.critic_loss[1])
# =================================================================== #
# Step 5: Setup the operations for computing model statistics. #
# =================================================================== #
# Setup the running means and standard deviations of the model inputs
# and outputs.
self.stats_ops, self.stats_names = self._setup_stats(scope or "Model")
def __init__(self,
sess,
ob_space,
ac_space,
co_space,
buffer_size,
batch_size,
actor_lr,
critic_lr,
verbose,
tau,
gamma,
use_huber,
l2_penalty,
model_params,
noise,
target_policy_noise,
target_noise_clip,
scope=None,
num_envs=1):
"""Instantiate the feed-forward neural network policy.
Parameters
----------
sess : tf.compat.v1.Session
the current TensorFlow session
ob_space : gym.spaces.*
the observation space of the environment
ac_space : gym.spaces.*
the action space of the environment
co_space : gym.spaces.*
the context space of the environment
buffer_size : int
the max number of transitions to store
batch_size : int
SGD batch size
actor_lr : float
actor learning rate
critic_lr : float
critic learning rate
verbose : int
the verbosity level: 0 none, 1 training information, 2 tensorflow
debug
tau : float
target update rate
gamma : float
discount factor
use_huber : bool
specifies whether to use the huber distance function as the loss
for the critic. If set to False, the mean-squared error metric is
used instead
l2_penalty : float
L2 regularization penalty. This is applied to the policy network.
model_params : dict
dictionary of model-specific parameters. See parent class.
noise : float
scaling term to the range of the action space, that is subsequently
used as the standard deviation of Gaussian noise added to the
action if `apply_noise` is set to True in `get_action`
target_policy_noise : float
standard deviation term to the noise from the output of the target
actor policy. See TD3 paper for more.
target_noise_clip : float
clipping term for the noise injected in the target actor policy
scope : str
an upper-level scope term. Used by policies that call this one.
"""
super(FeedForwardPolicy, self).__init__(
sess=sess,
ob_space=ob_space,
ac_space=ac_space,
co_space=co_space,
buffer_size=buffer_size,
batch_size=batch_size,
actor_lr=actor_lr,
critic_lr=critic_lr,
verbose=verbose,
tau=tau,
gamma=gamma,
use_huber=use_huber,
l2_penalty=l2_penalty,
model_params=model_params,
num_envs=num_envs,
)
# action magnitudes
ac_mag = 0.5 * (ac_space.high - ac_space.low)
self.noise = noise * ac_mag
self.target_policy_noise = np.array([ac_mag * target_policy_noise])
self.target_noise_clip = np.array([ac_mag * target_noise_clip])
# Compute the shape of the input observation space, which may include
# the contextual term.
ob_dim = self._get_ob_dim(ob_space, co_space)
# =================================================================== #
# Step 1: Create a replay buffer object. #
# =================================================================== #
self.replay_buffer = ReplayBuffer(
buffer_size=self.buffer_size,
batch_size=self.batch_size,
obs_dim=ob_dim[0],
ac_dim=self.ac_space.shape[0],
)
# =================================================================== #
# Step 2: Create input variables. #
# =================================================================== #
with tf.compat.v1.variable_scope("input", reuse=False):
self.terminals1 = tf.compat.v1.placeholder(tf.float32,
shape=(None, 1),
name='terminals1')
self.rew_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, 1),
name='rewards')
self.action_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) +
ac_space.shape,
name='actions')
self.obs_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + ob_dim,
name='obs0')
self.obs1_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + ob_dim,
name='obs1')
# =================================================================== #
# Step 3: Create actor and critic variables. #
# =================================================================== #
# Create networks and core TF parts that are shared across setup parts.
with tf.compat.v1.variable_scope("model", reuse=False):
self.actor_tf = self.make_actor(self.obs_ph)
self.critic_tf = [
self.make_critic(self.obs_ph,
self.action_ph,
scope="qf_{}".format(i)) for i in range(2)
]
self.critic_with_actor_tf = [
self.make_critic(self.obs_ph,
self.actor_tf,
reuse=True,
scope="qf_{}".format(i)) for i in range(2)
]
with tf.compat.v1.variable_scope("target", reuse=False):
# create the target actor policy
actor_target = self.make_actor(self.obs1_ph)
# smooth target policy by adding clipped noise to target actions
target_noise = tf.random.normal(tf.shape(actor_target),
stddev=self.target_policy_noise)
target_noise = tf.clip_by_value(target_noise,
-self.target_noise_clip,
self.target_noise_clip)
# clip the noisy action to remain in the bounds
noisy_actor_target = tf.clip_by_value(actor_target + target_noise,
self.ac_space.low,
self.ac_space.high)
# create the target critic policies
critic_target = [
self.make_critic(self.obs1_ph,
noisy_actor_target,
scope="qf_{}".format(i)) for i in range(2)
]
# Create the target update operations.
init, soft = self._setup_target_updates('model', 'target', scope, tau,
verbose)
self.target_init_updates = init
self.target_soft_updates = soft
# =================================================================== #
# Step 4: Setup the optimizers for the actor and critic. #
# =================================================================== #
with tf.compat.v1.variable_scope("Optimizer", reuse=False):
self._setup_actor_optimizer(scope)
self._setup_critic_optimizer(critic_target, scope)
# =================================================================== #
# Step 5: Setup the operations for computing model statistics. #
# =================================================================== #
# Setup the running means and standard deviations of the model inputs
# and outputs.
self.stats_ops, self.stats_names = self._setup_stats(scope or "Model")
def __init__(self,
sess,
ob_space,
ac_space,
co_space,
buffer_size,
batch_size,
actor_lr,
critic_lr,
verbose,
tau,
gamma,
use_huber,
l2_penalty,
model_params,
target_entropy,
scope=None,
num_envs=1):
"""Instantiate the feed-forward neural network policy.
Parameters
----------
sess : tf.compat.v1.Session
the current TensorFlow session
ob_space : gym.spaces.*
the observation space of the environment
ac_space : gym.spaces.*
the action space of the environment
co_space : gym.spaces.*
the context space of the environment
buffer_size : int
the max number of transitions to store
batch_size : int
SGD batch size
actor_lr : float
actor learning rate
critic_lr : float
critic learning rate
verbose : int
the verbosity level: 0 none, 1 training information, 2 tensorflow
debug
tau : float
target update rate
gamma : float
discount factor
use_huber : bool
specifies whether to use the huber distance function as the loss
for the critic. If set to False, the mean-squared error metric is
used instead
l2_penalty : float
L2 regularization penalty. This is applied to the policy network.
model_params : dict
dictionary of model-specific parameters. See parent class.
target_entropy : float
target entropy used when learning the entropy coefficient. If set
to None, a heuristic value is used.
scope : str
an upper-level scope term. Used by policies that call this one.
"""
super(FeedForwardPolicy, self).__init__(
sess=sess,
ob_space=ob_space,
ac_space=ac_space,
co_space=co_space,
buffer_size=buffer_size,
batch_size=batch_size,
actor_lr=actor_lr,
critic_lr=critic_lr,
verbose=verbose,
tau=tau,
gamma=gamma,
use_huber=use_huber,
l2_penalty=l2_penalty,
model_params=model_params,
num_envs=num_envs,
)
if target_entropy is None:
self.target_entropy = -np.prod(self.ac_space.shape)
else:
self.target_entropy = target_entropy
self._ac_means = 0.5 * (ac_space.high + ac_space.low)
self._ac_magnitudes = 0.5 * (ac_space.high - ac_space.low)
# Compute the shape of the input observation space, which may include
# the contextual term.
ob_dim = self._get_ob_dim(ob_space, co_space)
# =================================================================== #
# Step 1: Create a replay buffer object. #
# =================================================================== #
self.replay_buffer = ReplayBuffer(
buffer_size=self.buffer_size,
batch_size=self.batch_size,
obs_dim=ob_dim[0],
ac_dim=self.ac_space.shape[0],
)
# =================================================================== #
# Step 2: Create input variables. #
# =================================================================== #
with tf.compat.v1.variable_scope("input", reuse=False):
self.terminals1 = tf.compat.v1.placeholder(tf.float32,
shape=(None, 1),
name='terminals1')
self.rew_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, 1),
name='rewards')
self.action_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) +
ac_space.shape,
name='actions')
self.obs_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + ob_dim,
name='obs0')
self.obs1_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + ob_dim,
name='obs1')
# =================================================================== #
# Step 3: Create actor and critic variables. #
# =================================================================== #
# Create networks and core TF parts that are shared across setup parts.
with tf.compat.v1.variable_scope("model", reuse=False):
self.deterministic_action, self.policy_out, self.logp_pi, \
self.logp_action = self.make_actor(self.obs_ph, self.action_ph)
self.qf1, self.qf2, self.value_fn = self.make_critic(
self.obs_ph, self.action_ph, create_qf=True, create_vf=True)
self.qf1_pi, self.qf2_pi, _ = self.make_critic(self.obs_ph,
self.policy_out,
create_qf=True,
create_vf=False,
reuse=True)
# The entropy coefficient or entropy can be learned automatically,
# see Automating Entropy Adjustment for Maximum Entropy RL section
# of https://arxiv.org/abs/1812.05905
self.log_alpha = tf.compat.v1.get_variable('log_alpha',
dtype=tf.float32,
initializer=0.0)
self.alpha = tf.exp(self.log_alpha)
with tf.compat.v1.variable_scope("target", reuse=False):
# Create the value network
_, _, value_target = self.make_critic(self.obs1_ph,
create_qf=False,
create_vf=True)
self.value_target = value_target
# Create the target update operations.
init, soft = self._setup_target_updates('model/value_fns/vf',
'target/value_fns/vf', scope,
tau, verbose)
self.target_init_updates = init
self.target_soft_updates = soft
# =================================================================== #
# Step 4: Setup the optimizers for the actor and critic. #
# =================================================================== #
with tf.compat.v1.variable_scope("Optimizer", reuse=False):
self._setup_actor_optimizer(scope)
self._setup_critic_optimizer(scope)
# =================================================================== #
# Step 5: Setup the operations for computing model statistics. #
# =================================================================== #
# Setup the running means and standard deviations of the model inputs
# and outputs.
self.stats_ops, self.stats_names = self._setup_stats(scope or "Model")
def __init__(self,
sess,
ob_space,
ac_space,
co_space,
buffer_size,
batch_size,
learning_rate,
verbose,
layer_norm,
layers,
act_fun,
use_huber,
stochastic,
scope=None):
"""Instantiate the policy object.
Parameters
----------
sess : tf.compat.v1.Session
the current TensorFlow session
ob_space : gym.spaces.*
the observation space of the environment
ac_space : gym.spaces.*
the action space of the environment
co_space : gym.spaces.*
the context space of the environment
buffer_size : int
the max number of transitions to store
batch_size : int
SGD batch size
learning_rate : float
the learning rate for the policy
verbose : int
the verbosity level: 0 none, 1 training information, 2 tensorflow
debug
layer_norm : bool
enable layer normalisation
layers : list of int or None
the size of the Neural network for the policy
act_fun : tf.nn.*
the activation function to use in the neural network
use_huber : bool
specifies whether to use the huber distance function as the loss
function. If set to False, the mean-squared error metric is used
instead
stochastic : bool
specifies whether the policies are stochastic or deterministic
scope : str
an upper-level scope term. Used by policies that call this one.
"""
super(FeedForwardPolicy, self).__init__(sess=sess,
ob_space=ob_space,
ac_space=ac_space,
co_space=co_space,
buffer_size=buffer_size,
batch_size=batch_size,
learning_rate=learning_rate,
verbose=verbose,
layer_norm=layer_norm,
layers=layers,
act_fun=act_fun,
use_huber=use_huber,
stochastic=stochastic)
assert len(self.layers) >= 1, \
"Error: must have at least one hidden layer for the policy."
# Compute the shape of the input observation space, which may include
# the contextual term.
ob_dim = self._get_ob_dim(ob_space, co_space)
# =================================================================== #
# Step 1: Create a replay buffer object. #
# =================================================================== #
self.replay_buffer = ReplayBuffer(
buffer_size=self.buffer_size,
batch_size=self.batch_size,
obs_dim=ob_dim[0],
ac_dim=self.ac_space.shape[0],
)
# =================================================================== #
# Step 2: Create input variables. #
# =================================================================== #
with tf.compat.v1.variable_scope("input", reuse=False):
self.action_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) +
ac_space.shape,
name='actions')
self.obs_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + ob_dim,
name='obs0')
# =================================================================== #
# Step 3: Create policy variables. #
# =================================================================== #
self.policy = None
self.logp_ac = None
# Create networks and core TF parts that are shared across setup parts.
with tf.compat.v1.variable_scope("model", reuse=False):
if self.stochastic:
self._setup_stochastic_policy(self.obs_ph, self.action_ph)
else:
self._setup_deterministic_policy(self.obs_ph)
# =================================================================== #
# Step 4: Setup the optimizer. #
# =================================================================== #
self.loss = None
self.optimizer = None
with tf.compat.v1.variable_scope("Optimizer", reuse=False):
if self.stochastic:
self._setup_stochastic_optimizer(scope)
else:
self._setup_deterministic_optimizer(self.action_ph, scope)
# =================================================================== #
# Step 5: Setup the operations for computing model statistics. #
# =================================================================== #
# Setup the running means and standard deviations of the model inputs
# and outputs.
self.stats_ops, self.stats_names = self._setup_stats(scope or "Model")