def test_output_value(self, filters, in_channels, strides):
model = CNNModel(filters=filters,
strides=strides,
name='cnn_model',
padding='VALID',
hidden_w_init=tf.constant_initializer(1),
hidden_nonlinearity=None)
outputs = model.build(self._input_ph).outputs
output = self.sess.run(outputs,
feed_dict={self._input_ph: self.obs_input})
filter_sum = 1
# filter value after 3 layers of conv
for filter_iter, in_channel in zip(filters, in_channels):
filter_sum *= filter_iter[1][0] * filter_iter[1][1] * in_channel
height_size = self.input_height
width_size = self.input_width
for filter_iter, stride in zip(filters, strides):
height_size = int((height_size - filter_iter[1][0]) / stride) + 1
width_size = int((width_size - filter_iter[1][1]) / stride) + 1
flatten_shape = height_size * width_size * filters[-1][0]
# flatten
expected_output = np.full((self.batch_size, flatten_shape),
filter_sum,
dtype=np.float32)
assert np.array_equal(output, expected_output)
def test_is_pickleable(self, filters, strides):
model = CNNModel(filters=filters,
strides=strides,
name='cnn_model',
padding='VALID',
hidden_w_init=tf.constant_initializer(1),
hidden_nonlinearity=None)
outputs = model.build(self._input_ph).outputs
with tf.compat.v1.variable_scope('cnn_model/cnn/h0', reuse=True):
bias = tf.compat.v1.get_variable('bias')
bias.load(tf.ones_like(bias).eval())
output1 = self.sess.run(outputs,
feed_dict={self._input_ph: self.obs_input})
h = pickle.dumps(model)
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
model_pickled = pickle.loads(h)
# pylint: disable=unsubscriptable-object
input_shape = self.obs_input.shape[1:] # height, width, channel
input_ph = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) + input_shape,
name='input')
outputs = model_pickled.build(input_ph).outputs
output2 = sess.run(outputs, feed_dict={input_ph: self.obs_input})
assert np.array_equal(output1, output2)
def test_is_pickleable(self, filter_sizes, in_channels, out_channels,
strides):
model = CNNModel(filter_dims=filter_sizes,
num_filters=out_channels,
strides=strides,
name='cnn_model',
padding='VALID',
hidden_w_init=tf.constant_initializer(1),
hidden_nonlinearity=None)
outputs = model.build(self._input_ph)
with tf.variable_scope('cnn_model/cnn/h0', reuse=True):
bias = tf.get_variable('bias')
self.sess.run(tf.assign(bias, tf.ones_like(bias)))
output1 = self.sess.run(outputs,
feed_dict={self._input_ph: self.obs_input})
h = pickle.dumps(model)
with tf.Session(graph=tf.Graph()) as sess:
model_pickled = pickle.loads(h)
input_shape = self.obs_input.shape[1:] # height, width, channel
input_ph = tf.placeholder(tf.float32,
shape=(None, ) + input_shape,
name='input')
outputs = model_pickled.build(input_ph)
output2 = sess.run(outputs, feed_dict={input_ph: self.obs_input})
assert np.array_equal(output1, output2)
def __init__(self,
env_spec,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pad,
name='CategoricalCNNPolicy',
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.initializers.glorot_uniform(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.softmax,
output_w_init=tf.initializers.glorot_uniform(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
if not isinstance(env_spec.action_space, akro.Discrete):
raise ValueError(
'CategoricalCNNPolicy only works with akro.Discrete action '
'space.')
if not isinstance(env_spec.observation_space, akro.Box) or \
not len(env_spec.observation_space.shape) in (2, 3):
raise ValueError(
'{} can only process 2D, 3D akro.Image or'
' akro.Box observations, but received an env_spec with '
'observation_space of type {} and shape {}'.format(
type(self).__name__,
type(env_spec.observation_space).__name__,
env_spec.observation_space.shape))
super().__init__(name, env_spec)
self.obs_dim = env_spec.observation_space.shape
self.action_dim = env_spec.action_space.n
self.model = Sequential(
CNNModel(filter_dims=conv_filter_sizes,
num_filters=conv_filters,
strides=conv_strides,
padding=conv_pad,
hidden_nonlinearity=hidden_nonlinearity,
name='CNNModel'),
MLPModel(output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization,
name='MLPModel'))
self._initialize()
def __init__(self,
env_spec,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pad,
name='CategoricalConvPolicy',
hidden_sizes=[],
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.softmax,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
assert isinstance(env_spec.action_space, akro.Discrete), (
'CategoricalConvPolicy only works with akro.Discrete action '
'space.')
super().__init__(name, env_spec)
self.obs_dim = env_spec.observation_space.shape
self.action_dim = env_spec.action_space.n
self.model = Sequential(
CNNModel(
filter_dims=conv_filter_sizes,
num_filters=conv_filters,
strides=conv_strides,
padding=conv_pad,
hidden_nonlinearity=hidden_nonlinearity,
name='CNNModel'),
MLPModel(
output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization,
name='MLPModel'))
self._initialize()
def __init__(self,
env_spec,
filters,
strides,
hidden_sizes=(256, ),
name=None,
padding='SAME',
max_pooling=False,
pool_strides=(2, 2),
pool_shapes=(2, 2),
cnn_hidden_nonlinearity=tf.nn.relu,
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.initializers.glorot_uniform(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.initializers.glorot_uniform(),
output_b_init=tf.zeros_initializer(),
dueling=False,
layer_normalization=False):
if not isinstance(env_spec.observation_space, akro.Box) or \
not len(env_spec.observation_space.shape) in (2, 3):
raise ValueError(
'{} can only process 2D, 3D akro.Image or'
' akro.Box observations, but received an env_spec with '
'observation_space of type {} and shape {}'.format(
type(self).__name__,
type(env_spec.observation_space).__name__,
env_spec.observation_space.shape))
super().__init__(name)
self._env_spec = env_spec
self._action_dim = env_spec.action_space.n
self._filters = filters
self._strides = strides
self._hidden_sizes = hidden_sizes
self._padding = padding
self._max_pooling = max_pooling
self._pool_strides = pool_strides
self._pool_shapes = pool_shapes
self._cnn_hidden_nonlinearity = cnn_hidden_nonlinearity
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._layer_normalization = layer_normalization
self._dueling = dueling
self.obs_dim = self._env_spec.observation_space.shape
action_dim = self._env_spec.action_space.flat_dim
if not max_pooling:
cnn_model = CNNModel(filters=filters,
strides=strides,
padding=padding,
hidden_nonlinearity=cnn_hidden_nonlinearity)
else:
cnn_model = CNNModelWithMaxPooling(
filters=filters,
strides=strides,
padding=padding,
pool_strides=pool_strides,
pool_shapes=pool_shapes,
hidden_nonlinearity=cnn_hidden_nonlinearity)
if not dueling:
output_model = MLPModel(output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
else:
output_model = MLPDuelingModel(
output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self.model = Sequential(cnn_model, output_model)
self._initialize()
def __init__(self,
env_spec,
filter_dims,
num_filters,
strides,
hidden_sizes=[256],
name=None,
padding='SAME',
max_pooling=False,
pool_strides=(2, 2),
pool_shapes=(2, 2),
cnn_hidden_nonlinearity=tf.nn.relu,
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
dueling=False,
layer_normalization=False):
super().__init__(name)
self._env_spec = env_spec
self._action_dim = env_spec.action_space.n
self._filter_dims = filter_dims
self._num_filters = num_filters
self._strides = strides
self._hidden_sizes = hidden_sizes
self._padding = padding
self._max_pooling = max_pooling
self._pool_strides = pool_strides
self._pool_shapes = pool_shapes
self._cnn_hidden_nonlinearity = cnn_hidden_nonlinearity
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._layer_normalization = layer_normalization
self._dueling = dueling
self.obs_dim = self._env_spec.observation_space.shape
action_dim = self._env_spec.action_space.flat_dim
if not max_pooling:
cnn_model = CNNModel(filter_dims=filter_dims,
num_filters=num_filters,
strides=strides,
padding=padding,
hidden_nonlinearity=cnn_hidden_nonlinearity)
else:
cnn_model = CNNModelWithMaxPooling(
filter_dims=filter_dims,
num_filters=num_filters,
strides=strides,
padding=padding,
pool_strides=pool_strides,
pool_shapes=pool_shapes,
hidden_nonlinearity=cnn_hidden_nonlinearity)
if not dueling:
output_model = MLPModel(output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
else:
output_model = MLPDuelingModel(
output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self.model = Sequential(cnn_model, output_model)
self._initialize()