class DDPGCriticNetwork(LayerBasedNetwork):
def __init__(self,
scope='ddpg-critic-network',
summary_labels=(),
size_t0=400,
size_t1=300):
super(DDPGCriticNetwork, self).__init__(scope=scope,
summary_labels=summary_labels)
self.t0l = Linear(size=size_t0, scope='linear0')
self.t0b = TFLayer(layer='batch_normalization',
scope='batchnorm0',
center=True,
scale=True)
self.t0n = Nonlinearity(name='relu', scope='relu0')
self.t1l = Linear(size=size_t1, scope='linear1')
self.t1b = TFLayer(layer='batch_normalization',
scope='batchnorm1',
center=True,
scale=True)
self.t1n = Nonlinearity(name='relu', scope='relu1')
self.t2d = Dense(size=1,
activation='tanh',
scope='dense0',
weights=tf.random_uniform_initializer(minval=-3e-3,
maxval=3e-3))
self.add_layer(self.t0l)
self.add_layer(self.t0b)
self.add_layer(self.t0n)
self.add_layer(self.t1l)
self.add_layer(self.t1b)
self.add_layer(self.t1n)
self.add_layer(self.t2d)
def tf_apply(self, x, internals, update, return_internals=False):
assert x['states'], x['actions']
if isinstance(x['states'], dict):
if len(x['states']) != 1:
raise TensorForceError(
'DDPG critic network must have only one state input, but {} given.'
.format(len(x['states'])))
x_states = x['states'][next(iter(sorted(x['states'])))]
else:
x_states = x['states']
if isinstance(x['actions'], dict):
if len(x['actions']) != 1:
raise TensorForceError(
'DDPG critic network must have only one action input, but {} given.'
.format(len(x['actions'])))
x_actions = x['actions'][next(iter(sorted(x['actions'])))]
else:
x_actions = x['actions']
out = self.t0l.apply(x=x_states, update=update)
out = self.t0b.apply(x=out, update=update)
out = self.t0n.apply(x=out, update=update)
out = self.t1l.apply(x=tf.concat([out, x_actions], axis=1),
update=update)
out = self.t1b.apply(x=out, update=update)
out = self.t1n.apply(x=out, update=update)
out = self.t2d.apply(x=out, update=update)
# Remove last dimension because we only return Q values for one state and action
# out = tf.squeeze(out)
if return_internals:
# Todo: Internals management
return out, None
else:
return out
class EIIE_OutPut(Layer):
"""
EIIE Output layer
based on 2-dimensional convolutional layer.
use of two entries:
- * : the precedent treatments
- last_w : the last weights coming from environment
"""
def __init__(self,
l2_regularization=0.0,
l1_regularization=0.0,
scope='eieeoutput',
summary_labels=()):
"""
2D convolutional layer.
Args:
size: Number of filters set to 1
window: Convolution window size, either an integer or pair of integers. calculated
stride: Convolution stride, either an integer or pair of integers.
padding: Convolution padding, one of 'VALID' or 'SAME'
bias: If true, a bias is added
activation: Type of nonlinearity, or dict with name & arguments
l2_regularization: L2 regularization weight
l1_regularization: L1 regularization weight
"""
self.size = 1
self.stride = 1
self.padding = 'VALID'
self.bias = True
activation = 'relu'
self.l2_regularization = l2_regularization
self.l1_regularization = l1_regularization
self.nonlinearity = Nonlinearity(name=activation,
summary_labels=summary_labels)
super(EIIE_OutPut, self).__init__(scope=scope,
summary_labels=summary_labels)
def tf_apply(self, x, update):
if util.rank(x) != 4:
raise TensorForceError(
'Invalid input rank for conv2d layer: {}, must be 4'.format(
util.rank(x)))
self.window = (1, x.shape[2])
filters_shape = self.window + (x.shape[3].value, self.size)
stddev = min(0.1, sqrt(2.0 / self.size))
filters_init = tf.random_normal_initializer(mean=0.0,
stddev=stddev,
dtype=tf.float32)
self.filters = tf.get_variable(name='W',
shape=filters_shape,
dtype=tf.float32,
initializer=filters_init)
stride_h, stride_w = self.stride if type(self.stride) is tuple else (
self.stride, self.stride)
x = tf.nn.conv2d(input=x,
filter=self.filters,
strides=(1, stride_h, stride_w, 1),
padding=self.padding)
if self.bias:
bias_shape = (self.size, )
bias_init = tf.zeros_initializer(dtype=tf.float32)
self.bias = tf.get_variable(name='b',
shape=bias_shape,
dtype=tf.float32,
initializer=bias_init)
x = tf.nn.bias_add(value=x, bias=self.bias)
x = self.nonlinearity.apply(x=x, update=update)
if 'activations' in self.summary_labels:
summary = tf.summary.histogram(name='activations', values=x)
self.summaries.append(summary)
return x
def tf_regularization_loss(self):
regularization_loss = super(EIIE_OutPut, self).tf_regularization_loss()
if regularization_loss is None:
losses = list()
else:
losses = [regularization_loss]
if self.l2_regularization > 0.0:
losses.append(self.l2_regularization *
tf.nn.l2_loss(t=self.filters))
if self.bias is not None:
losses.append(self.l2_regularization *
tf.nn.l2_loss(t=self.bias))
if self.l1_regularization > 0.0:
losses.append(self.l1_regularization *
tf.reduce_sum(input_tensor=tf.abs(x=self.filters)))
if self.bias is not None:
losses.append(self.l1_regularization *
tf.reduce_sum(input_tensor=tf.abs(x=self.bias)))
regularization_loss = self.nonlinearity.regularization_loss()
if regularization_loss is not None:
losses.append(regularization_loss)
if len(losses) > 0:
return tf.add_n(inputs=losses)
else:
return None
def get_variables(self, include_non_trainable=False):
layer_variables = super(
EIIE_OutPut,
self).get_variables(include_non_trainable=include_non_trainable)
nonlinearity_variables = self.nonlinearity.get_variables(
include_non_trainable=include_non_trainable)
return layer_variables + nonlinearity_variables
def get_summaries(self):
layer_summaries = super(EIIE_OutPut, self).get_summaries()
nonlinearity_summaries = self.nonlinearity.get_summaries()
return layer_summaries + nonlinearity_summaries
