def conv2d_encoder(x,
layer_config=(
(32, (3, 1), (2, 1)),
(32, (3, 1), (2, 1)),
(32, (3, 1), (2, 1)),
),
pad='SAME',
name='encoder',
reuse=False):
"""
Defines convolutional encoder.
Args:
x: input tensor
layer_config: first to last nested layers configuration list: [layer_1_config, layer_2_config,...], where:
layer_i_config = [num_filters(int), filter_size(list), stride(list)]
pad: str, padding scheme: 'SAME' or 'VALID'
name: str, mame scope
reuse: bool
Returns:
list of tensors holding encoded features for every layer outer to inner,
level-wise list of encoding layers shapes, first ro last.
"""
with tf.variable_scope(name, reuse=reuse):
layer_shapes = [x.get_shape()]
layer_outputs = []
for i, layer_spec in enumerate(layer_config, 1):
x = tf.nn.elu(
norm_layer(
conv2d(
x=x,
num_filters=layer_spec[0],
name='/conv_kernels_{}'.format(i ),
filter_size=layer_spec[1],
stride=layer_spec[2],
pad=pad,
reuse=reuse
)
),
name='encoder_layer_{}'.format(i),
)
layer_shapes.append(x.get_shape())
layer_outputs.append(x)
return layer_outputs, layer_shapes
def conv_2d_network(x,
ob_space,
ac_space,
conv_2d_layer_ref=conv2d,
conv_2d_num_filters=(32, 32, 64, 64),
conv_2d_filter_size=(3, 3),
conv_2d_stride=(2, 2),
pad="SAME",
dtype=tf.float32,
name='conv2d',
collections=None,
reuse=False,
**kwargs):
"""
Stage1 network: from preprocessed 2D input to estimated features.
Encapsulates convolutions + layer normalisation + nonlinearity. Can be shared.
Returns:
tensor holding state features;
"""
with tf.variable_scope(name, reuse=reuse):
#for i in range(conv_2d_num_layers):
for i, num_filters in enumerate(conv_2d_num_filters):
x = tf.nn.elu(
norm_layer(
conv_2d_layer_ref(
x,
num_filters,
"_layer_{}".format(i + 1),
conv_2d_filter_size,
conv_2d_stride,
pad,
dtype,
collections,
reuse
),
scope=name + "_layer_{}".format(i + 1)
)
)
# A3c/BaseAAC original paper design:
#x = tf.nn.elu(conv2d(x, 16, 'conv2d_1', [8, 8], [4, 4], pad, dtype, collections, reuse))
#x = tf.nn.elu(conv2d(x, 32, 'conv2d_2', [4, 4], [2, 2], pad, dtype, collections, reuse))
#x = tf.nn.elu(
# linear(batch_flatten(x), 256, 'conv_2d_dense', normalized_columns_initializer(0.01), reuse=reuse)
#)
return x
def dense_aac_network(x, ac_space, name='dense_aac', linear_layer_ref=noisy_linear, reuse=False):
"""
Stage3 network: from LSTM flattened output to advantage actor-critic.
Returns:
logits tensor
value function tensor
action sampling function.
"""
with tf.variable_scope(name, reuse=reuse):
# Center-logits:
logits = norm_layer(
linear_layer_ref(
x=x,
size=ac_space,
name='action',
initializer=normalized_columns_initializer(0.01),
reuse=reuse
),
center=True,
scale=False,
)
# logits = linear_layer_ref(
# x=x,
# size=ac_space,
# name='action',
# initializer=normalized_columns_initializer(0.01),
# reuse=reuse
# )
vf = tf.reshape(
linear_layer_ref(
x=x,
size=1,
name="value",
initializer=normalized_columns_initializer(1.0),
reuse=reuse
),
[-1]
)
sample = categorical_sample(logits, ac_space)[0, :]
return logits, vf, sample
def conv_1d_casual_attention_encoder(
x,
keep_prob,
ob_space=None,
ac_space=None,
conv_1d_num_filters=32,
conv_1d_filter_size=2,
conv_1d_activation=tf.nn.elu,
conv_1d_attention_ref=tf.contrib.seq2seq.LuongAttention,
# conv_1d_attention_ref=tf.contrib.seq2seq.BahdanauAttention,
name='casual_encoder',
reuse=False,
collections=None,
**kwargs):
"""
Tree-shaped convolution stack encoder with self-attention.
Stage1 casual convolutions network: from 1D input to estimated features.
Returns:
tensor holding state features;
"""
with tf.variable_scope(name_or_scope=name, reuse=reuse):
shape = x.get_shape().as_list()
if len(shape) > 3: # remove pseudo 2d dimension
x = x[:, :, 0, :]
num_layers = int(math.log(shape[1], conv_1d_filter_size))
# print('num_layers: ', num_layers)
layers = []
attention_layers = []
y = x
for i in range(num_layers):
_, length, channels = y.get_shape().as_list()
# t2b:
tail = length % conv_1d_filter_size
if tail != 0:
pad = conv_1d_filter_size - tail
paddings = [[0, 0], [pad, 0], [0, 0]]
y = tf.pad(y, paddings)
length += pad
# print('padded_length: ', length)
num_time_batches = int(length / conv_1d_filter_size)
# print('num_time_batches: ', num_time_batches)
y = tf.reshape(y, [-1, conv_1d_filter_size, channels],
name='layer_{}_t2b'.format(i))
y = conv1d(x=y,
num_filters=conv_1d_num_filters,
filter_size=conv_1d_filter_size,
stride=1,
pad='VALID',
name='conv1d_layer_{}'.format(i))
y = tf.nn.dropout(y, keep_prob=keep_prob)
# b2t:
y = tf.reshape(y, [-1, num_time_batches, conv_1d_num_filters],
name='layer_{}_output'.format(i))
y = norm_layer(y)
if conv_1d_activation is not None:
y = conv_1d_activation(y)
layers.append(y)
# Insert attention for all but top layer:
if num_time_batches > 1:
attention = attention_layer(
y,
attention_ref=conv_1d_attention_ref,
name='attention_layer_{}'.format(i))
attention_layers.append(attention)
convolved = tf.stack([h[:, -1, :] for h in layers],
axis=1,
name='convolved_stack')
attended = tf.concat(attention_layers, axis=-2, name='attention_stack')
encoded = tf.concat([convolved, attended],
axis=-2,
name='encoded_state')
# print('convolved: ', convolved)
# print('attention_stack: ', attended)
# print('encoded :', encoded)
return encoded
def conv_1d_casual_encoder(
x,
ob_space,
ac_space,
conv_1d_num_filters=32,
conv_1d_filter_size=2,
conv_1d_activation=tf.nn.elu,
conv_1d_overlap=1,
name='casual_encoder',
keep_prob=None,
conv_1d_gated=False,
reuse=False,
collections=None,
**kwargs
):
"""
Tree-shaped convolution stack encoder as more comp. efficient alternative to dilated one.
Stage1 casual convolutions network: from 1D input to estimated features.
Returns:
tensor holding state features;
"""
with tf.variable_scope(name_or_scope=name, reuse=reuse):
shape = x.get_shape().as_list()
if len(shape) > 3: # remove pseudo 2d dimension
x = x[:, :, 0, :]
num_layers = int(math.log(shape[1], conv_1d_filter_size))
# print('num_layers: ', num_layers)
layers = []
slice_depth = []
y = x
for i in range(num_layers):
_, length, channels = y.get_shape().as_list()
# t2b:
tail = length % conv_1d_filter_size
if tail != 0:
pad = conv_1d_filter_size - tail
paddings = [[0, 0], [pad, 0], [0, 0]]
y = tf.pad(y, paddings)
length += pad
# print('padded_length: ', length)
num_time_batches = int(length / conv_1d_filter_size)
# print('num_time_batches: ', num_time_batches)
y = tf.reshape(y, [-1, conv_1d_filter_size, channels], name='layer_{}_t2b'.format(i))
y = conv1d(
x=y,
num_filters=conv_1d_num_filters,
filter_size=conv_1d_filter_size,
stride=1,
pad='VALID',
name='conv1d_layer_{}'.format(i)
)
# b2t:
y = tf.reshape(y, [-1, num_time_batches, conv_1d_num_filters], name='layer_{}_output'.format(i))
y = norm_layer(y)
if conv_1d_activation is not None:
y = conv_1d_activation(y)
if keep_prob is not None:
y = tf.nn.dropout(y, keep_prob=keep_prob, name="_layer_{}_with_dropout".format(i))
layers.append(y)
depth = conv_1d_overlap // conv_1d_filter_size ** i
if depth < 1:
depth = 1
slice_depth.append(depth)
# encoded = tf.stack([h[:, -1, :] for h in layers], axis=1, name='encoded_state')
sliced_layers = [
tf.slice(
h,
begin=[0, h.get_shape().as_list()[1] - d, 0],
size=[-1, d, -1]
) for h, d in zip(layers, slice_depth)
]
output_stack = sliced_layers
# But:
if conv_1d_gated:
split_size = int(conv_1d_num_filters / 2)
output_stack = []
for l in sliced_layers:
x1 = l[..., :split_size]
x2 = l[..., split_size:]
y = tf.multiply(
x1,
tf.nn.sigmoid(x2),
name='gated_conv_output'
)
output_stack.append(y)
encoded = tf.concat(output_stack, axis=1, name='encoded_state')
# encoded = tf.concat(
# [
# tf.slice(
# h,
# begin=[0, h.get_shape().as_list()[1] - d, 0],
# size=[-1, d, -1]
# ) for h, d in zip(layers, slice_depth)
# ],
# axis=1,
# name='encoded_state'
# )
print('encoder :', encoded)
return encoded