def FractalNet(image, n_tools, n_recs, blocks=[64], **kwargs):
'''
- blocks: a list, ordered from network in to out, of each block's n_chan
'''
x = layers.Conv2D(blocks[0], 1, 1, activation='relu')(image) # embedding
for n_chan in blocks:
x = FractalBlock(x, n_recs, n_chan, **kwargs)
act = layers.Conv2D(n_tools, 1, 1, activation='relu')(x)
act = conv_to_fc(act)
val = layers.Conv2D(1, 1, 1, activation='relu')(x)
val = conv_to_fc(val)
return act, val
def nature_cnn(scaled_images, **kwargs):
"""
CNN from Nature paper.
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=4,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_3)
return activ(linear(layer_3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def __init__(self, sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=False, **kwargs):
super(NatureCNN, self).__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=reuse, scale=True)
with tf.variable_scope("model", reuse=reuse):
activ = tf.nn.relu
input = self.processed_obs
layer_1 = activ(conv(input, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs))
layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs))
layer_3 = activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs))
layer_3 = conv_to_fc(layer_3)
extracted_features = activ(linear(layer_3, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
value_fn = tf.layers.dense(extracted_features, 1, name='vf')
self.proba_distribution, self.policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(extracted_features, extracted_features, init_scale=0.01)
self.value_fn = value_fn
self.initial_state = None
self._setup_init()
total = 0
for v in tf.trainable_variables():
dims = v.get_shape().as_list()
num = int(np.prod(dims))
total += num
print(' %s \t\t Num: %d \t\t Shape %s ' % (v.name, num, dims))
print('\nTotal number of params: %d' % total)
def ppo_cnn(scaled_images, **kwargs):
activ = tf.nn.elu
conv1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=5,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
conv2 = activ(
conv(conv1,
'c2',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
conv3 = activ(
conv(conv2,
'c3',
n_filters=64,
filter_size=3,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
conv3 = conv_to_fc(conv3)
return activ(linear(conv3, 'fc1', n_hidden=512, init_scale=0.01))
def minigrid_extractor_small(scaled_images, **kwargs):
"""
CNN for MiniGrid environments with variable grid sizes
"""
activ = tf.nn.relu
# first layer is just an embedding finder
layer_1 = conv(scaled_images,
'c1',
n_filters=32,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs)
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_4 = conv_to_fc(layer_3)
print(layer_3)
return activ(linear(layer_4, 'fc1', n_hidden=128, init_scale=np.sqrt(2)))
def sac_cnn_lstm(scaled_images, **kwargs):
activ = tf.nn.relu
conv1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=5,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
conv2 = activ(
conv(conv1,
'c2',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
conv3 = activ(
conv(conv2,
'c3',
n_filters=64,
filter_size=3,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
conv3 = conv_to_fc(conv3)
# try w/o LSTM first
return activ(linear(conv3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def __init__(self, sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=False, **kwargs):
super(CNNPolicy, self).__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=reuse, scale=True)
with tf.variable_scope("model", reuse=reuse):
# activ = tf.nn.relu
# extracted_features = nature_cnn(self.processed_obs, **kwargs)
# extracted_features = tf.layers.flatten(extracted_features)
activ = tf.nn.relu
self.n1 = activ(conv(self.processed_obs, 'c1', n_filters=32, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs))
self.n2 = activ(conv(self.n1, 'c2', n_filters=64, filter_size=3, stride=2, init_scale=np.sqrt(2), **kwargs))
self.n3 = activ(conv(self.n2, 'c3', n_filters=64, filter_size=2, stride=2, init_scale=np.sqrt(2), **kwargs))
self.flattened_n3 = conv_to_fc(self.n3)
pi_h = self.flattened_n3
for i, layer_size in enumerate([512]):
pi_h = activ(tf.layers.dense(pi_h, layer_size, name='pi_fc' + str(i)))
pi_latent = pi_h
vf_h = pi_latent
value_fn = tf.layers.dense(vf_h, 1, name='vf')
vf_latent = vf_h
self._proba_distribution, self._policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(pi_latent, vf_latent, init_scale=0.01)
self._value_fn = value_fn
self._setup_init()
def Cnn1(image, **kwargs):
activ = tf.nn.relu
layer_1 = activ(
conv(image,
'c1',
n_filters=32,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_3)
return activ(linear(layer_3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def modified_cnn(scaled_images, **kwargs):
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=8,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=16,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=32,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_3)
return activ(
linear(layer_3, 'fc1', n_hidden=144, init_scale=np.sqrt(2)))
def ValShrink(val, **kwargs):
activ = tf.nn.relu
val = activ(
conv(val,
'v1',
n_filters=64,
filter_size=3,
stride=2,
init_scale=np.sqrt(2)))
val = activ(
conv(val,
'v2',
n_filters=64,
filter_size=3,
stride=2,
init_scale=np.sqrt(3)))
#val = activ(conv(val, 'v3', n_filters=64, filter_size=3, stride=2,
# init_scale=np.sqrt(2)))
val = activ(
conv(val,
'v4',
n_filters=64,
filter_size=1,
stride=1,
init_scale=np.sqrt(2)))
val = conv_to_fc(val)
return val
def dynamics(scaled_images, action, **kwargs):
"""
Dynamic function
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c3',
n_filters=16,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c4',
n_filters=32,
filter_size=4,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_2)
layer_4 = tf.concat(values=[action, layer_3], axis=-1)
return tf.nn.sigmoid(
linear(layer_4, 'fc2', n_hidden=256, init_scale=np.sqrt(2)))
def cnn_3d(scaled_voxels, n_hidden, filters, filter_sizes, strides, **kwargs):
"""
CNN in 3D.
:param scaled_voxels: (TensorFlow Tensor) Voxel input placeholder
:param n_hidden: (int) Number of nodes in the last linear layer
:param filters: (array) Filter numbers for the convolutional layers of the CNN
:param filter_sizes: (array) Filter sizes for the convolutional layers of the CNN
:param strides: (array) Strides for the convolutional layers of the CNN
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.tanh
layers = []
for i, (n_filter, filter_size,
stride) in enumerate(zip(filters, filter_sizes, strides)):
input_layer = scaled_voxels if i == 0 else layers[-1]
label = 'c%d' % (i + 1)
layer = activ(
conv3d(input_layer,
label,
n_filters=n_filter,
filter_size=filter_size,
stride=stride,
init_scale=np.sqrt(2),
**kwargs))
layers.append(layer)
print('layer_%d' % (i + 1), layer.shape)
layer = conv_to_fc(layers[-1])
return tf.tanh(
linear(layer, 'fc1', n_hidden=n_hidden, init_scale=np.sqrt(2)))
def custom_cnn(scaled_images, **kwargs):
"""
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=(1, 4),
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=(1, 3),
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_2)
return activ(linear(layer_3, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
def cnn(scaled_images, **kwargs):
activ = tf.nn.relu
l1 = activ(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=2,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
l2 = activ(
conv(l1,
'c2',
n_filters=64,
filter_size=2,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
l3 = activ(
conv(l2,
'c3',
n_filters=64,
filter_size=2,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
l4 = conv_to_fc(l3)
l5 = activ(linear(l4, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
return l5
def sequence_1d_cnn_ego_bypass_tc(scaled_sequence, **kwargs):
"""
CNN for sequence.
:param scaled_sequence: (TensorFlow Tensor) Image input placeholder
:shape of scaled_sequence: (Batch, Time, Channels)
:scaled_sequence => [norm_ego_speed] + [loop_back..current][relative_leading_norm_s, relative following_norm_s...]
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
norm_ego = scaled_sequence[:, -1:, :1]
norm_ego = tf.reshape(norm_ego, [-1, 1])
fc2_ego = norm_ego
relative_others = scaled_sequence[:, :, 1:]
layer_1_others = activ(
conv1d(relative_others,
'c1_others',
n_filters=32,
filter_size=2,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_2_others = conv_to_fc(layer_1_others)
layer_3_others = activ(
linear(layer_2_others, 'fc2', n_hidden=256, init_scale=np.sqrt(2)))
concat_out = tf.concat([fc2_ego, layer_3_others], axis=1, name='concat')
return activ(linear(concat_out, 'fc3', n_hidden=256,
init_scale=np.sqrt(2)))
def tic_tac_toe_cnn(scaled_images, **kwargs):
"""
Custom CNN for Tic Tac Toe env.
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer = scaled_images
# print(kwargs)
net_arch = kwargs['cnn_arch']
filter_size = kwargs['filter_size']
pad = kwargs['pad']
for i, f in enumerate(net_arch[:-1], start=1):
# print('c' + str(i), f)
layer = activ(conv(layer, 'c' + str(i), n_filters=f, filter_size=filter_size,
stride=1, pad=pad, data_format='NCHW'))
layer = conv_to_fc(layer)
# print('fc1', net_arch[-1])
# print()
return activ(linear(layer, 'fc1', n_hidden=net_arch[-1]))
def __init__(self,
sess,
ob_space,
ac_space,
n_env,
n_steps,
n_batch,
reuse=False,
**kwargs):
super(NatureCNN, self).__init__(sess,
ob_space,
ac_space,
n_env,
n_steps,
n_batch,
reuse=reuse,
scale=True)
with tf.variable_scope("model", reuse=reuse):
activ = tf.nn.relu
input = self.processed_obs
layer_1 = activ(
conv(input,
'c1',
n_filters=32,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=64,
filter_size=4,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
layer_3 = conv_to_fc(layer_3)
extracted_features = activ(
linear(layer_3, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
value_fn = tf.layers.dense(extracted_features, 1, name='vf')
self.proba_distribution, self.policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(extracted_features, extracted_features, init_scale=0.01)
self.value_fn = value_fn
self.initial_state = None
self._setup_init()
def modified_cnn(unscaled_images, **kwargs): import tensorflow as tf scaled_images = tf.cast(unscaled_images, tf.float32) / 255. activ = tf.nn.relu layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=1, stride=1, init_scale=np.sqrt(2), **kwargs)) layer_2 = activ(conv(layer_1, 'c2', n_filters=32, filter_size=2, stride=2, init_scale=np.sqrt(2), **kwargs)) layer_2 = conv_to_fc(layer_2) return activ(linear(layer_2, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def modified_cnn(scaled_images, **kwargs): import tensorflow as tf activ = tf.nn.relu layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)) layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)) layer_3 = activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)) layer_3 = conv_to_fc(layer_3) return activ(linear(layer_3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def my_small_cnn(scaled_images, **kwargs):
activ = tf.nn.relu
layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=3,
stride=1, **kwargs))
layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=3,
stride=1, **kwargs))
layer_3 = conv_to_fc(layer_2)
return activ(
linear(layer_3, 'fc1', n_hidden=32, init_scale=np.sqrt(2)))
def attention_cnn(scaled_images, **kwargs):
"""Nature CNN with region-sensitive module"""
def softmax_2d(tensor):
b, h, w, c = tensor.shape
tensor = tf.reshape(tensor, (-1, h * w, c))
tensor = tf.nn.softmax(tensor, axis=1)
tensor = tf.reshape(tensor, (-1, h, w, c))
return tensor
c1 = tf.nn.relu(
conv(scaled_images,
'c1',
n_filters=32,
filter_size=8,
stride=4,
init_scale=np.sqrt(2),
**kwargs))
c2 = tf.nn.relu(
conv(c1,
'c2',
n_filters=64,
filter_size=4,
stride=2,
init_scale=np.sqrt(2),
**kwargs))
c3 = tf.nn.relu(
conv(c2,
'c3',
n_filters=64,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
c3 = tf.nn.l2_normalize(c3, axis=-1)
a1 = tf.nn.elu(
conv(c3,
'a1',
n_filters=512,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
a2 = softmax_2d(
conv(a1,
'a2',
n_filters=2,
filter_size=1,
stride=1,
init_scale=np.sqrt(2),
**kwargs))
a2 = tf.identity(a2, name='attn')
x = c3 * tf.reduce_sum(a2, axis=-1, keepdims=True)
x = conv_to_fc(x)
return tf.nn.relu(linear(x, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def __init__(self, sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=False, **kwargs):
super(ONet, self).__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=reuse, scale=True)
with tf.variable_scope("model", reuse=reuse):
activ = tf.nn.relu
input = self.processed_obs
with tf.variable_scope("CNNlayers"):
with tf.variable_scope("CNNLayer1"):
layer_11 = activ(conv(input, 'c11', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_12 = activ(conv(layer_11, 'c12', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_13 = tf.nn.max_pool(layer_12, name="p13", ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
with tf.variable_scope("CNNLayer2"):
layer_21 = activ(conv(layer_13, 'c21', n_filters=128, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_22 = activ(conv(layer_21, 'c22', n_filters=128, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_23 = tf.nn.max_pool(layer_22, name="p23", ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
with tf.variable_scope("CNNLayer3"):
layer_31 = activ(conv(layer_23, 'c31', n_filters=256, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_32 = activ(conv(layer_31, 'c32', n_filters=256, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_33 = activ(conv(layer_32, 'c33', n_filters=256, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_34 = tf.nn.max_pool(layer_33, name="p34", ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
with tf.variable_scope("CNNLayer4"):
layer_41 = activ(conv(layer_34, 'c41', n_filters=512, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_42 = activ(conv(layer_41, 'c42', n_filters=512, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_43 = activ(conv(layer_42, 'c43', n_filters=512, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_44 = tf.nn.max_pool(layer_43, name="p44", ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
with tf.variable_scope("CNNLayer5"):
layer_51 = activ(conv(layer_44, 'c51', n_filters=512, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_52 = activ(conv(layer_51, 'c52', n_filters=512, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_53 = activ(conv(layer_52, 'c53', n_filters=512, filter_size=3, stride=1, init_scale=np.sqrt(2), pad="SAME", **kwargs))
layer_54 = tf.nn.max_pool(layer_53, name="p54", ksize=[1, 8, 8, 1], strides=[1, 8, 8, 1], padding='SAME')
with tf.variable_scope("DenseLayer6"):
layer_61 = conv_to_fc(layer_54)
layer_62 = activ(linear(layer_61, 'fc62', n_hidden=128, init_scale=np.sqrt(2)))
layer_63 = activ(linear(layer_62, 'fc63', n_hidden=64, init_scale=np.sqrt(2)))
#layer_64 = activ(linear(layer_63, 'fc64', n_hidden=64, init_scale=np.sqrt(2)))
layer_64 = layer_63
value_fn = tf.layers.dense(layer_64, 1, name='vf')
self.proba_distribution, self.policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(layer_64, layer_64, init_scale=0.01)
total = 0
for v in tf.trainable_variables():
dims = v.get_shape().as_list()
num = int(np.prod(dims))
total += num
print(' %s \t\t Num: %d \t\t Shape %s ' % (v.name, num, dims))
print('\nTotal number of params: %d' % total)
self.value_fn = value_fn
self.initial_state = None
self._setup_init()
def cnn_custom(image, **kwargs):
"""
CNN feature extrator for 2048.
:param image: (TensorFlow Tensor) Image input placeholder.
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN.
:return: (TensorFlow Tensor) The CNN output layer.
"""
activ = tf.nn.relu
layer_1 = activ(
conv(image,
'c1',
n_filters=128,
filter_size=4,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=128,
filter_size=3,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=256,
filter_size=2,
stride=2,
pad='VALID',
init_scale=np.sqrt(2),
**kwargs))
layer_4 = activ(
conv(layer_3,
'c4',
n_filters=256,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_5 = activ(
conv(layer_4,
'c5',
n_filters=512,
filter_size=2,
stride=1,
pad='VALID',
init_scale=np.sqrt(2),
**kwargs))
layer_lin = conv_to_fc(layer_5)
return layer_lin
def cnn_extractor(scaled_images, channels=c, w=w, h=h):
print(f"========= REAL SHAPE: {scaled_images.shape} ===========")
original_shape = scaled_images.shape[1]
print(f"========= SHAPE: {original_shape} ===========")
scaled_images = tf.reshape(scaled_images, (-1, h, w, channels))
activ = tf.nn.relu
layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=w, stride=1, init_scale=np.sqrt(2)))
layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=1, stride=1, init_scale=np.sqrt(2)))
layer_3 = activ(conv(layer_2, 'c3', n_filters=128, filter_size=1, stride=1, init_scale=np.sqrt(2)))
layer_3 = conv_to_fc(layer_3)
return activ(linear(layer_3, 'fc1', n_hidden=128, init_scale=np.sqrt(2)))
def cnn_5l4(image, **kwargs):
"""
:param in: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
layer_1 = activ(
conv(image,
'c1',
n_filters=222,
filter_size=4,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=222,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=222,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_4 = activ(
conv(layer_3,
'c4',
n_filters=222,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_5 = activ(
conv(layer_4,
'c5',
n_filters=222,
filter_size=2,
stride=1,
pad='SAME',
init_scale=np.sqrt(2),
**kwargs))
layer_lin = conv_to_fc(layer_5)
return layer_lin
def custom_cnn(scaled_images, **kwargs):
activ = tf.tanh
shape = scaled_images.shape
if len(shape) == 4:
njets = scaled_images.shape[1]
filter_width = scaled_images.shape[2]
if len(shape) == 3:
njets = 1
filter_width = scaled_images.shape[1]
scaled_images = tf.reshape(scaled_images, [-1, njets, filter_width, 2])
n_filters = arch[0]
# first conv
layer = activ(
conv(scaled_images,
'c1',
n_filters=n_filters,
filter_size=(1, filter_width),
stride=1,
init_scale=init_scale,
**kwargs))
# shape it so that we can apply another conv
layer = tf.reshape(layer, [-1, njets, n_filters, 1])
n_filters_prev = n_filters
for i, n_filters in enumerate(arch):
# pass first layer
if i == 0:
continue
# apply conv
layer = activ(
conv(layer,
'c' + str(i + 1),
n_filters=n_filters,
filter_size=(1, n_filters_prev),
stride=1,
init_scale=init_scale,
**kwargs))
# shape it so that we can apply another conv
layer = tf.reshape(layer, [-1, njets, n_filters, 1])
# get n_filters of current layer for the next layer
n_filters_prev = n_filters
# get back to a flat tensor of size n_jet
last_layer = conv_to_fc(layer)
return last_layer
def nature_cnn(scaled_images, **kwargs):
"""
CNN from Nature paper.
:param scaled_images: (TensorFlow Tensor) Image input placeholder
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
activ = tf.nn.relu
print(scaled_images)
# scaled_images = tf.transpose(scaled_images, [0, 3, 1, 2])
# print(scaled_images)
layer_1 = activ(
conv(scaled_images,
'c1',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
print('layer_1', layer_1)
layer_2 = activ(
conv(layer_1,
'c2',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
print('layer_2', layer_2)
layer_3 = activ(
conv(layer_2,
'c3',
n_filters=256,
filter_size=3,
stride=1,
init_scale=np.sqrt(2),
pad='SAME',
**kwargs))
print('layer_3', layer_3)
# layer_4 = activ(
# conv(layer_3, 'c4', n_filters=256, filter_size=3, stride=1, init_scale=np.sqrt(2), pad='SAME', **kwargs))
# layer_5 = activ(
# conv(layer_4, 'c5', n_filters=256, filter_size=3, stride=1, init_scale=np.sqrt(2), pad='SAME', **kwargs))
layer_5 = conv_to_fc(layer_3)
return activ(linear(layer_5, 'fc1', n_hidden=256, init_scale=np.sqrt(2)))
def mobilenet_cnn(images, **kwargs):
activ = tf.nn.relu
#TODO: try this: https://github.com/HongyangGao/ChannelNets/blob/master/model.py
cur_out_num = 32
data_format = kwargs.get('data_format', 'NHWC')
outs = ops.conv2d(images,
cur_out_num, (3, 3),
'conv_s',
train=False,
stride=2,
act_fn=None,
data_format=data_format)
cur_outs = ops.dw_block( # 112 * 112 * 64
outs,
cur_out_num,
1,
'conv_1_0',
1.0,
False,
data_format=data_format)
outs = tf.concat([outs, cur_outs], axis=-1, name='add0')
cur_out_num *= 2
outs = ops.dw_block( # 56 * 56 * 128
outs,
cur_out_num,
2,
'conv_1_1',
1.0,
False,
data_format=data_format)
cur_outs = ops.dw_block( # 56 * 56 * 128
outs,
cur_out_num,
1,
'conv_1_2',
1.0,
False,
data_format=data_format)
outs = tf.concat([outs, cur_outs], axis=-1, name='add1')
outs = ops.dw_block( # 7 * 7 * 1024
outs,
cur_out_num,
1,
'conv_3_1',
1.0,
False,
data_format=data_format)
layer_3 = conv_to_fc(outs)
return activ(linear(layer_3, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def navigation_cnn(observation, **kwargs):
"""
Modified CNN (Nature).
:param observation: (TensorFlow Tensor) Image and relative distance to goal input placeholders
:param kwargs: (dict) Extra keywords parameters for the convolutional layers of the CNN
:return: (TensorFlow Tensor) The CNN output layer
"""
# np.shape(observation) = (1,84,85,1)
print("Shape observation: ", np.shape(observation))
scaled_images = observation[:, :, 0:-1, :]
# With LIDAR
scalar = observation[:, :, -1, :]
# navigation_info = scalar[:, :, 0] # Reshape in order to concatenate the arrays
# WITHOUT LIDAR
navigation_info = scalar[:, 0:3, :] # Uncomment if you don't want use the lidar
navigation_info = navigation_info[:, :, 0] # Reshape in order to concatenate the arrays
# TODO: navigation_info needs to be normalized in [0,1] like the scaled images
# navigation_info = navigation_info * 255 / 3.14 # Denormalize the vector multiplying by ob_space.high and normalise on the bearing.high (3.14)
print("Scaled images: ", np.shape(scaled_images))
print("Scalar: ", np.shape(scalar))
print("NavigationInfo: ", np.shape(navigation_info))
activ = tf.nn.elu
layer_1 = norm_layer(activ(conv(scaled_images, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs)), 'c1_norm')
print("Layer1: ", np.shape(layer_1))
layer_2 = norm_layer(activ(conv(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs)), 'c2_norm')
print("Layer2: ", np.shape(layer_2))
layer_3 = norm_layer(activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)), 'c3_norm')
print("Layer3: ", np.shape(layer_3))
# layer_1 = (activ(conv(scaled_images, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs)))
# layer_2 = (activ(conv(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs)))
# layer_3 = (activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs)))
# layer_1 = activ(conv(scaled_images, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs))
# layer_2 = activ(conv(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs))
# layer_3 = activ(conv(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs))
layer_3 = conv_to_fc(layer_3)
print("Layer3: ", np.shape(layer_3))
#layer_3 = tf.nn.sigmoid(layer_3) # To squeeze values in [0,1]
#print("L3: ", np.shape(layer_3))
#print("NI: ", np.shape(navigation_info))
fc_1 = tf.concat([layer_3, navigation_info], axis=1)
#print("L3: ", np.shape(layer_3))
# filter_summaries = tf.summary.merge([tf.summary.image("raw_observation", scaled_images, max_outputs=32),
# tf.summary.image("filters/conv1", layer_1,
# max_outputs=32)])
return layer_3, tf.nn.relu(linear(fc_1, 'fc1', n_hidden=512, init_scale=np.sqrt(2)))
def __init__(self, sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=False, **kwargs):
super(MultiInputPolicy, self).__init__(sess, ob_space, ac_space, n_env, n_steps, n_batch, reuse=reuse, scale=False)
with tf.variable_scope("model", reuse=reuse):
#input = tf.placeholder(shape=self.processed_obs.shape,dtype=self.processed_obs.dtype )
depth = self.processed_obs[n_batch:, :, :(settings.encoded_depth_H * settings.encoded_depth_W)]
pos = self.processed_obs[n_batch:, :, (settings.encoded_depth_H * settings.encoded_depth_W):]
if(n_batch == None):
depth = tf.reshape(depth, shape=(-1, settings.encoded_depth_H, settings.encoded_depth_W, 1))
pos = tf.reshape(pos, shape=(-1,1,settings.position_depth))
else:
depth = tf.reshape(depth, shape=(n_batch, settings.encoded_depth_H, settings.encoded_depth_W, 1))
pos = tf.reshape(pos, shape=(n_batch, 1, settings.position_depth))
# Convolutions on Depth Images
activ = tf.nn.relu
layer_1 = activ(conv_grey(depth, 'c1', n_filters=32, filter_size=8, stride=4, init_scale=np.sqrt(2), **kwargs))
layer_2 = activ(conv_grey(layer_1, 'c2', n_filters=64, filter_size=4, stride=2, init_scale=np.sqrt(2), **kwargs))
layer_3 = activ(conv_grey(layer_2, 'c3', n_filters=64, filter_size=3, stride=1, init_scale=np.sqrt(2), **kwargs))
layer_3 = conv_to_fc(layer_3)
image_encoded = tf.keras.layers.Flatten()(layer_3)
# Fully connected layers for pos vector
pos_layer_1 = tf.keras.layers.Dense(32, activation='relu', name= 'pos_fc_1')(pos)
pos_layer_2 = tf.keras.layers.Dense(32, activation='relu', name= 'pos_fc_2')(pos_layer_1)
pos_encoded = tf.keras.layers.Flatten()(pos_layer_2)
joint_encoding = tf.keras.layers.concatenate([image_encoded, pos_encoded])
x = tf.keras.layers.Dense(64, activation="tanh", name='pi_fc_0')(joint_encoding)
pi_latent = tf.keras.layers.Dense(64, activation="tanh", name='pi_fc_1')(x)
x1 = tf.keras.layers.Dense(64, activation="tanh", name='vf_fc_0')(joint_encoding)
vf_latent = tf.keras.layers.Dense(64, activation="tanh", name='vf_fc_1')(x1)
value_fn = tf.keras.layers.Dense(1, name='vf')(vf_latent)
self.proba_distribution, self.policy, self.q_value = \
self.pdtype.proba_distribution_from_latent(pi_latent, vf_latent, init_scale=0.01)
self.value_fn = value_fn
self.initial_state = None
self._setup_init()
