def __init__(self, sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=False, **kwargs):
super(NatureCNN, self).__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=reuse, scale=True)
with tf.variable_scope("model", reuse=reuse):
activ = tf.nn.relu
input = self.processed_obs
layer_1 = activ(conv(input, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs))
layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs))
layer_3 = activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs))
layer_3 = conv_to_fc(layer_3)
extracted_features = activ(linear(layer_3, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
value_fn = tf.layers.dense(extracted_features, 1, name='vf')
self.proba_distribution, self.policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(extracted_features, extracted_features, init_scale=0.01)
self.value_fn = value_fn
self.initial_state = None
self._setup_init()
total = 0
for v in tf.trainable_variables():
dims = v.get_shape().as_list()
num = int(np.prod(dims))
total += num
print(' %s \t\t Num: %d \t\t Shape %s ' % (v.name, num, dims))
print('\nTotal number of params: %d' % total)
def res_block(input, res_scope=None, num=None, **kwargs):
activ = tf.nn.relu
output_temp = conv(input,
res_scope + '_temp',
n_filters=num,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs)
output_temp = tf.layers.batch_normalization(output_temp,
name=res_scope + '_temp_bn',
training=False)
output_temp = activ(output_temp)
output = conv(output_temp,
res_scope,
n_filters=num,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs)
output = tf.layers.batch_normalization(output,
name=res_scope + '_bn',
training=False)
output = tf.add(output, input)
output = activ(output)
return output
def ppo_cnn(scaled_images, **kwargs):
activ = tf.nn.elu
conv1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=5,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
conv2 = activ(
conv(conv1,
'c2',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
conv3 = activ(
conv(conv2,
'c3',
n_filters=64,
filter_size=3,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
conv3 = conv_to_fc(conv3)
return activ(linear(conv3, 'fc1', n_hidden=512, init_scale=0.01))
def concise_cnn(scaled_images, **kwargs):
"""
input = [H,W,D] H=W
output = [H,W,64]
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
return layer_2
def modified_cnn(scaled_images, **kwargs):
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=8,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=16,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=32,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_3)
return activ(
linear(layer_3, 'fc1', n_hidden=144, init_scale=np.sqrt(2)))
def sac_cnn_lstm(scaled_images, **kwargs):
activ = tf.nn.relu
conv1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=5,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
conv2 = activ(
conv(conv1,
'c2',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
conv3 = activ(
conv(conv2,
'c3',
n_filters=64,
filter_size=3,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
conv3 = conv_to_fc(conv3)
# try w/o LSTM first
return activ(linear(conv3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def custom_cnn(scaled_images, params, **kwargs):
"""
Custom CNN Architecture builder for this project
"""
try:
activ = getattr(tf.nn, params['activ'])
except Exception as e:
print(e, 'Invalid activation function.')
init_scale = params['conv_init_scale']
# First layer
out = activ(
conv(input_tensor=scaled_images,
scope='c0',
n_filters=params['filters'][0],
filter_size=params['kernel_size'][0],
stride=params['stride'][0],
init_scale=init_scale,
**kwargs))
# Following layers
for i, layer in enumerate(params['filters'][1:]):
out = activ(
conv(input_tensor=out,
scope='c{}'.format(i + 1),
n_filters=layer,
filter_size=params['kernel_size'][i + 1],
stride=params['stride'][i + 1],
init_scale=init_scale,
**kwargs))
n_hidden = np.prod([v.value for v in out.get_shape()[1:]])
out = tf.reshape(out, [-1, n_hidden])
return out
def simple_cnn(scaled_images, **kwargs):
"""
simple CNN, input = [14*4,14*4,C].
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
print('scaled_images', scaled_images)
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=12,
filter_size=2,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=24,
filter_size=2,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
print('layer_2', layer_2)
return layer_2
def Cnn1(image, **kwargs):
activ = tf.nn.relu
layer_1 = activ(
conv(image,
'c1',
n_filters=32,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_3)
return activ(linear(layer_3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def ValShrink(val, **kwargs):
activ = tf.nn.relu
val = activ(
conv(val,
'v1',
n_filters=64,
filter_size=3,
stride=2,
init_scale=np.sqrt(2)))
val = activ(
conv(val,
'v2',
n_filters=64,
filter_size=3,
stride=2,
init_scale=np.sqrt(3)))
#val = activ(conv(val, 'v3', n_filters=64, filter_size=3, stride=2,
# init_scale=np.sqrt(2)))
val = activ(
conv(val,
'v4',
n_filters=64,
filter_size=1,
stride=1,
init_scale=np.sqrt(2)))
val = conv_to_fc(val)
return val
def dynamics(scaled_images, action, **kwargs):
"""
Dynamic function
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c3',
n_filters=16,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c4',
n_filters=32,
filter_size=4,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_2)
layer_4 = tf.concat(values=[action, layer_3], axis=-1)
return tf.nn.sigmoid(
linear(layer_4, 'fc2', n_hidden=256, init_scale=np.sqrt(2)))
def cnn(scaled_images, **kwargs):
activ = tf.nn.relu
l1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=2,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
l2 = activ(
conv(l1,
'c2',
n_filters=64,
filter_size=2,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
l3 = activ(
conv(l2,
'c3',
n_filters=64,
filter_size=2,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
l4 = conv_to_fc(l3)
l5 = activ(linear(l4, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
return l5
def custom_cnn(scaled_images, **kwargs):
"""
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=(1, 4),
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=(1, 3),
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_2)
return activ(linear(layer_3, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
def minigrid_extractor_small(scaled_images, **kwargs):
"""
CNN for MiniGrid environments with variable grid sizes
"""
activ = tf.nn.relu
# first layer is just an embedding finder
layer_1 = conv(scaled_images,
'c1',
n_filters=32,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs)
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_4 = conv_to_fc(layer_3)
print(layer_3)
return activ(linear(layer_4, 'fc1', n_hidden=128, init_scale=np.sqrt(2)))
def __init__(self, sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=False, **kwargs):
super(CNNPolicy, self).__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=reuse, scale=True)
with tf.variable_scope("model", reuse=reuse):
# activ = tf.nn.relu
# extracted_features = nature_cnn(self.processed_obs, **kwargs)
# extracted_features = tf.layers.flatten(extracted_features)
activ = tf.nn.relu
self.n1 = activ(conv(self.processed_obs, 'c1', n_filters=32, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs))
self.n2 = activ(conv(self.n1, 'c2', n_filters=64, filter_size=3, stride=2, init_scale=np.sqrt(2), **kwargs))
self.n3 = activ(conv(self.n2, 'c3', n_filters=64, filter_size=2, stride=2, init_scale=np.sqrt(2), **kwargs))
self.flattened_n3 = conv_to_fc(self.n3)
pi_h = self.flattened_n3
for i, layer_size in enumerate([512]):
pi_h = activ(tf.layers.dense(pi_h, layer_size, name='pi_fc' + str(i)))
pi_latent = pi_h
vf_h = pi_latent
value_fn = tf.layers.dense(vf_h, 1, name='vf')
vf_latent = vf_h
self._proba_distribution, self._policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(pi_latent, vf_latent, init_scale=0.01)
self._value_fn = value_fn
self._setup_init()
def nature_cnn(scaled_images, **kwargs):
"""
CNN from Nature paper.
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=4,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_3)
return activ(linear(layer_3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def __init__(self,
sess,
ob_space,
ac_space,
n_env,
n_steps,
n_batch,
reuse=False,
**kwargs):
super(NatureCNN, self).__init__(sess,
ob_space,
ac_space,
n_env,
n_steps,
n_batch,
reuse=reuse,
scale=True)
with tf.variable_scope("model", reuse=reuse):
activ = tf.nn.relu
input = self.processed_obs
layer_1 = activ(
conv(input,
'c1',
n_filters=32,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=4,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_3)
extracted_features = activ(
linear(layer_3, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
value_fn = tf.layers.dense(extracted_features, 1, name='vf')
self.proba_distribution, self.policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(extracted_features, extracted_features, init_scale=0.01)
self.value_fn = value_fn
self.initial_state = None
self._setup_init()
def modified_cnn(unscaled_images, **kwargs): import tensorflow as tf scaled_images = tf.cast(unscaled_images, tf.float32) / 255. activ = tf.nn.relu layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=1, stride=1, init_scale=np.sqrt(2), **kwargs)) layer_2 = activ(conv(layer_1, 'c2', n_filters=32, filter_size=2, stride=2, init_scale=np.sqrt(2), **kwargs)) layer_2 = conv_to_fc(layer_2) return activ(linear(layer_2, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def modified_cnn(scaled_images, **kwargs): import tensorflow as tf activ = tf.nn.relu layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)) layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)) layer_3 = activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)) layer_3 = conv_to_fc(layer_3) return activ(linear(layer_3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def my_small_cnn(scaled_images, **kwargs):
activ = tf.nn.relu
layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=3,
stride=1, **kwargs))
layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=3,
stride=1, **kwargs))
layer_3 = conv_to_fc(layer_2)
return activ(
linear(layer_3, 'fc1', n_hidden=32, init_scale=np.sqrt(2)))
def attention_cnn(scaled_images, **kwargs):
"""Nature CNN with region-sensitive module"""
def softmax_2d(tensor):
b, h, w, c = tensor.shape
tensor = tf.reshape(tensor, (-1, h * w, c))
tensor = tf.nn.softmax(tensor, axis=1)
tensor = tf.reshape(tensor, (-1, h, w, c))
return tensor
c1 = tf.nn.relu(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
c2 = tf.nn.relu(
conv(c1,
'c2',
n_filters=64,
filter_size=4,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
c3 = tf.nn.relu(
conv(c2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
c3 = tf.nn.l2_normalize(c3, axis=-1)
a1 = tf.nn.elu(
conv(c3,
'a1',
n_filters=512,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
a2 = softmax_2d(
conv(a1,
'a2',
n_filters=2,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
a2 = tf.identity(a2, name='attn')
x = c3 * tf.reduce_sum(a2, axis=-1, keepdims=True)
x = conv_to_fc(x)
return tf.nn.relu(linear(x, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def cnn_custom(image, **kwargs):
"""
CNN feature extrator for 2048.
:param image: (TensorFlow Tensor) Image input placeholder.
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN.
:return: (TensorFlow Tensor) The CNN output layer.
"""
activ = tf.nn.relu
layer_1 = activ(
conv(image,
'c1',
n_filters=128,
filter_size=4,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=128,
filter_size=3,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=256,
filter_size=2,
stride=2,
pad='VALID',
init_scale=np.sqrt(2),
**kwargs))
layer_4 = activ(
conv(layer_3,
'c4',
n_filters=256,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_5 = activ(
conv(layer_4,
'c5',
n_filters=512,
filter_size=2,
stride=1,
pad='VALID',
init_scale=np.sqrt(2),
**kwargs))
layer_lin = conv_to_fc(layer_5)
return layer_lin
def cnn_5l4(image, **kwargs):
"""
:param in: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(image,
'c1',
n_filters=222,
filter_size=4,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=222,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=222,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_4 = activ(
conv(layer_3,
'c4',
n_filters=222,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_5 = activ(
conv(layer_4,
'c5',
n_filters=222,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_lin = conv_to_fc(layer_5)
return layer_lin
def cnn_extractor(scaled_images, channels=c, w=w, h=h):
print(f"========= REAL SHAPE: {scaled_images.shape} ===========")
original_shape = scaled_images.shape[1]
print(f"========= SHAPE: {original_shape} ===========")
scaled_images = tf.reshape(scaled_images, (-1, h, w, channels))
activ = tf.nn.relu
layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=w, stride=1, init_scale=np.sqrt(2)))
layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=1, stride=1, init_scale=np.sqrt(2)))
layer_3 = activ(conv(layer_2, 'c3', n_filters=128, filter_size=1, stride=1, init_scale=np.sqrt(2)))
layer_3 = conv_to_fc(layer_3)
return activ(linear(layer_3, 'fc1', n_hidden=128, init_scale=np.sqrt(2)))
def nature_cnn(scaled_images, last_hidden=200, **kwargs):
"""
CNN from Nature paper.
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
def convfc(input_tensor, vardim=None):
"""
Reshapes a Tensor from a convolutional network to a Tensor for a fully connected network
:param input_tensor: (TensorFlow Tensor) The convolutional input tensor
:return: (TensorFlow Tensor) The fully connected output tensor
"""
n_hidden = np.prod([v.value for v in input_tensor.get_shape()[1:]])
if vardim is not None:
input_tensor = tf.reshape(input_tensor, [vardim, n_hidden])
else:
input_tensor = tf.reshape(input_tensor, [-1, n_hidden])
return input_tensor
vardim = tf.shape(scaled_images)[0]
scaled_images = tf.reshape(scaled_images[:vardim, :12288],
[vardim, 64, 64, 3]) / 255.
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=32,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=32,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = convfc(layer_3, vardim)
return activ(
linear(layer_3, 'fc1', n_hidden=last_hidden, init_scale=np.sqrt(2)))
def custom_cnn(scaled_images, **kwargs):
activ = tf.tanh
shape = scaled_images.shape
if len(shape) == 4:
njets = scaled_images.shape[1]
filter_width = scaled_images.shape[2]
if len(shape) == 3:
njets = 1
filter_width = scaled_images.shape[1]
scaled_images = tf.reshape(scaled_images, [-1, njets, filter_width, 2])
n_filters = arch[0]
# first conv
layer = activ(
conv(scaled_images,
'c1',
n_filters=n_filters,
filter_size=(1, filter_width),
stride=1,
init_scale=init_scale,
**kwargs))
# shape it so that we can apply another conv
layer = tf.reshape(layer, [-1, njets, n_filters, 1])
n_filters_prev = n_filters
for i, n_filters in enumerate(arch):
# pass first layer
if i == 0:
continue
# apply conv
layer = activ(
conv(layer,
'c' + str(i + 1),
n_filters=n_filters,
filter_size=(1, n_filters_prev),
stride=1,
init_scale=init_scale,
**kwargs))
# shape it so that we can apply another conv
layer = tf.reshape(layer, [-1, njets, n_filters, 1])
# get n_filters of current layer for the next layer
n_filters_prev = n_filters
# get back to a flat tensor of size n_jet
last_layer = conv_to_fc(layer)
return last_layer
def nature_cnn(scaled_images, **kwargs):
"""
CNN from Nature paper.
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
print(scaled_images)
# scaled_images = tf.transpose(scaled_images, [0, 3, 1, 2])
# print(scaled_images)
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
print('layer_1', layer_1)
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
print('layer_2', layer_2)
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
print('layer_3', layer_3)
# layer_4 = activ(
# conv(layer_3, 'c4', n_filters=256, filter_size=3, stride=1, init_scale=np.sqrt(2), pad='SAME', **kwargs))
# layer_5 = activ(
# conv(layer_4, 'c5', n_filters=256, filter_size=3, stride=1, init_scale=np.sqrt(2), pad='SAME', **kwargs))
layer_5 = conv_to_fc(layer_3)
return activ(linear(layer_5, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
def navigation_cnn(observation, **kwargs):
"""
Modified CNN (Nature).
:param observation: (TensorFlow Tensor) Image and relative distance to goal input placeholders
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
# np.shape(observation) = (1,84,85,1)
print("Shape observation: ", np.shape(observation))
scaled_images = observation[:, :, 0:-1, :]
# With LIDAR
scalar = observation[:, :, -1, :]
# navigation_info = scalar[:, :, 0] # Reshape in order to concatenate the arrays
# WITHOUT LIDAR
navigation_info = scalar[:, 0:3, :] # Uncomment if you don't want use the lidar
navigation_info = navigation_info[:, :, 0] # Reshape in order to concatenate the arrays
# TODO: navigation_info needs to be normalized in [0,1] like the scaled images
# navigation_info = navigation_info * 255 / 3.14 # Denormalize the vector multiplying by ob_space.high and normalise on the bearing.high (3.14)
print("Scaled images: ", np.shape(scaled_images))
print("Scalar: ", np.shape(scalar))
print("NavigationInfo: ", np.shape(navigation_info))
activ = tf.nn.elu
layer_1 = norm_layer(activ(conv(scaled_images, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs)), 'c1_norm')
print("Layer1: ", np.shape(layer_1))
layer_2 = norm_layer(activ(conv(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs)), 'c2_norm')
print("Layer2: ", np.shape(layer_2))
layer_3 = norm_layer(activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)), 'c3_norm')
print("Layer3: ", np.shape(layer_3))
# layer_1 = (activ(conv(scaled_images, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs)))
# layer_2 = (activ(conv(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs)))
# layer_3 = (activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)))
# layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs))
# layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs))
# layer_3 = activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs))
layer_3 = conv_to_fc(layer_3)
print("Layer3: ", np.shape(layer_3))
#layer_3 = tf.nn.sigmoid(layer_3) # To squeeze values in [0,1]
#print("L3: ", np.shape(layer_3))
#print("NI: ", np.shape(navigation_info))
fc_1 = tf.concat([layer_3, navigation_info], axis=1)
#print("L3: ", np.shape(layer_3))
# filter_summaries = tf.summary.merge([tf.summary.image("raw_observation", scaled_images, max_outputs=32),
# tf.summary.image("filters/conv1", layer_1,
# max_outputs=32)])
return layer_3, tf.nn.relu(linear(fc_1, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def tic_tac_toe_cnn(scaled_images, **kwargs):
"""
Custom CNN for Tic Tac Toe env.
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer = scaled_images
# print(kwargs)
net_arch = kwargs['cnn_arch']
filter_size = kwargs['filter_size']
pad = kwargs['pad']
for i, f in enumerate(net_arch[:-1], start=1):
# print('c' + str(i), f)
layer = activ(conv(layer, 'c' + str(i), n_filters=f, filter_size=filter_size,
stride=1, pad=pad, data_format='NCHW'))
layer = conv_to_fc(layer)
# print('fc1', net_arch[-1])
# print()
return activ(linear(layer, 'fc1', n_hidden=net_arch[-1]))
def custom_cnn(scaled_images, **kwargs):
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
layer_21 = activ(
conv(layer_2,
'c21',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
layer_3 = activ(
conv(layer_21,
'c3',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
layer_4 = conv_to_fc(layer_3)
return activ(linear(layer_4, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
