def _build_convs(self, inputs, name):
self.conv1 = layers.conv2d(
inputs=inputs,
data_format="NHWC",
num_outputs=16,
kernel_size=5,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
scope="%s/conv1" % name,
trainable=self.trainable
)
self.conv2 = layers.conv2d(
inputs=self.conv1,
data_format="NHWC",
num_outputs=32,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
scope="%s/conv2" % name,
trainable=self.trainable
)
if self.trainable:
layers.summarize_activation(self.conv1)
layers.summarize_activation(self.conv2)
return self.conv2
def build_fully_connected_layers_with_batch_norm(the_input, shape, kernel_initializer, mode, scope_prefix=""):
"""
Builds fully connected layers with batch normalization onto the computational graph of the desired shape.
The following are a few examples of the shapes that can be given, and how the layers are built
shape = [512, 256, 128]
result: input --> 512 --> 256 --> 128
shape = [[25, 75], [200], 100, 75]
result: concat((input --> 25 --> 75), (input --> 200)) --> 100 --> 75
More complicated shapes can be used by defining modules recursively like the following
shape = [[[100], [50, 50]], [200, 50], [75], 100, 20]
"""
if len(shape) == 0:
return the_input
module_outputs = []
for j, inner_modules in enumerate(shape):
if isinstance(inner_modules, list):
output = build_fully_connected_layers_with_batch_norm(
the_input,
inner_modules,
kernel_initializer,
mode,
scope_prefix="%sfc_module_%d/" % (scope_prefix, j + 1))
module_outputs.append(output)
else:
if len(module_outputs) == 1:
the_input = module_outputs[0]
elif len(module_outputs) > 1:
the_input = tf.concat(module_outputs, axis=1)
for i, layer_shape in enumerate(shape[j:]):
with tf.variable_scope(scope_prefix + "FC_" + str(i + 1)):
pre_activation = tf.layers.dense(
inputs=the_input,
units=layer_shape,
use_bias=False,
kernel_initializer=kernel_initializer(),
name="layer")
batch_normalized = tf.layers.batch_normalization(pre_activation,
scale=False,
training=(mode == tf.estimator.ModeKeys.TRAIN),
fused=True)
the_input = tf.nn.relu(batch_normalized)
layers.summarize_activation(the_input)
module_outputs = [the_input]
break
if len(module_outputs) == 1:
return module_outputs[0]
return tf.concat(module_outputs, axis=1)
def _build_convs(inputs, name):
"""
helper method for building screen and minimap conv networks
"""
conv1 = layers.conv2d(
inputs=inputs,
data_format="NHWC",
num_outputs=16,
kernel_size=5,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
scope="%s/conv1" % name
)
conv2 = layers.conv2d(
inputs=conv1,
data_format="NHWC",
num_outputs=32,
kernel_size=3,
stride=1,
padding='SAME',
activation_fn=tf.nn.relu,
scope="%s/conv2" % name
)
layers.summarize_activation(conv1)
layers.summarize_activation(conv2)
return conv2
def build_conv_layers_for_input(self, inputs, name):
"""build_conv_layers_for_input
Creates 2 convolutional layers based on an input.
Changeable parts here are:
Number of outputs for both layers
Size of the kernel used
The stride used
The activation function
:param inputs: The inputs to run the convolutional layers against.
:param name: The name of the input, to scope the layers.
"""
conv_layer1 = layers.conv2d(inputs=inputs,
data_format="NHWC",
num_outputs=16,
kernel_size=5,
stride=1,
padding="SAME",
activation_fn=tf.nn.relu,
scope="%s/conv_layer1" % name,
trainable=self.trainable)
conv_layer2 = layers.conv2d(inputs=conv_layer1,
data_format="NHWC",
num_outputs=32,
kernel_size=3,
stride=1,
padding="SAME",
activation_fn=tf.nn.relu,
scope="%s/conv_layer2" % name,
trainable=self.trainable)
if self.trainable:
layers.summarize_activation(conv_layer1)
layers.summarize_activation(conv_layer2)
tf.summary.image(f"{name}/conv_layer1",
tf.reshape(conv_layer1, [-1, 32, 32, 1]), 3)
tf.summary.image(f"{name}/conv_layer2",
tf.reshape(conv_layer2, [-1, 32, 32, 1]), 3)
return conv_layer2
def _build_convs(self, inputs, name):
conv1 = layers.conv2d(
inputs=inputs,
data_format="NHWC",
num_outputs=32,
kernel_size=8,
stride=4,
padding='SAME',
activation_fn=tf.nn.relu,
scope="%s/conv1" % name,
trainable=self.trainable
)
conv2 = layers.conv2d(
inputs=conv1,
data_format="NHWC",
num_outputs=64,
kernel_size=4,
stride=1,#2,#
padding='SAME',
activation_fn=tf.nn.relu,
scope="%s/conv2" % name,
trainable=self.trainable
)
# conv3 = layers.conv2d(
# inputs=conv2,
# data_format="NHWC",
# num_outputs=64,
# kernel_size=3,
# stride=1,
# padding='SAME',
# activation_fn=tf.nn.relu,
# scope="%s/conv3" % name,
# trainable=self.trainable
# )
if self.trainable:
layers.summarize_activation(conv1)
layers.summarize_activation(conv2)
# layers.summarize_activation(conv3)
return conv2
def build_transposed_inception_module_with_batch_norm(
the_input,
module,
kernel_initializer,
mode,
activation_summaries=[],
num_previously_built_inception_modules=0,
padding='same',
force_no_concat=False,
make_trainable=True,
weight_regularizer=None):
if weight_regularizer is None:
weight_regularizer = lambda: None
path_outputs = [None for _ in range(len(module))]
to_summarize = []
cur_input = None
for j, path in enumerate(module):
with tf.variable_scope("inception_module_" +
str(num_previously_built_inception_modules +
1) + "_path_" + str(j + 1)):
for i, section in enumerate(path):
if i == 0:
if j != 0:
path_outputs[j - 1] = cur_input
cur_input = the_input
cur_conv_output = tf.layers.conv2d_transpose(
inputs=cur_input,
filters=section[0],
kernel_size=section[1],
strides=section[2],
padding=padding,
use_bias=False,
activation=None,
kernel_initializer=kernel_initializer(),
kernel_regularizer=weight_regularizer(),
trainable=make_trainable)
# name="layer_" + str(i + 1))
cur_batch_normalized = tf.layers.batch_normalization(
cur_conv_output,
training=(mode == tf.estimator.ModeKeys.TRAIN),
trainable=make_trainable,
fused=True)
cur_input = tf.nn.relu(cur_batch_normalized)
to_summarize.append(cur_input)
path_outputs[-1] = cur_input
activation_summaries = activation_summaries + [
layers.summarize_activation(layer) for layer in to_summarize
]
with tf.variable_scope("inception_module_" +
str(num_previously_built_inception_modules + 1)):
if len(path_outputs) == 1:
return path_outputs[0], activation_summaries
for j in range(1, len(path_outputs)):
if force_no_concat or path_outputs[0].get_shape().as_list(
)[1:3] != path_outputs[j].get_shape().as_list()[1:3]:
return [temp_input
for temp_input in path_outputs], activation_summaries
return tf.concat([temp_input for temp_input in path_outputs],
3), activation_summaries
def build_convolutional_module_with_batch_norm(the_input, module, kernel_initializer, mode,
num_previously_built_inception_modules=0, make_trainable=True,
weight_regularizer=None, data_format="NHWC"):
"""
Builds a convolutional module based on a given design using batch normalization and the rectifier activation.
It returns the final layer/layers in the module.
The following are a few examples of what can be used in the 'module' parameter (explanation follows):
example_1_module = [[[35,1], (1024, 8)]]
example_2_module = [[[30, 1]],
[[15, 1], [30, 3]],
[[15, 1], [30, 3, 2]],
[[15, 1], [30, 3, 3]], # <-- This particular path does a 1x1 convolution on the module's
[[10, 1], [20, 3, 4]], # input with 15 filters, followed by a 3x3 'same' padded
[[8, 1], [16, 3, 5]], # convolution with dilation factor of 3. It is concatenated
[[8, 1], [16, 2, (2, 4)]], # with the output of the other paths, and then returned
[[8, 1], [16, 2, (4, 2)]]]
:param module: A list (representing the module's shape), of lists (each representing the shape of a 'path' from the
input to the output of the module), of either tuples or lists of size 2 or 3 (representing individual layers).
If a tuple is used, it indicates that the layer should use 'valid' padding, and if a list is used it will use a
padding of 'same'. The contents of the innermost list or tuple will be the number of filters to create for a layer,
followed by the information to pass to conv2d as kernel_size, and then optionally, a third element which is
to be passed to conv2d as a dilation factor.
"""
if weight_regularizer is None:
weight_regularizer = lambda:None
path_outputs = [None for _ in range(len(module))]
to_summarize = []
cur_input = None
for j, path in enumerate(module):
with tf.variable_scope("module_" + str(num_previously_built_inception_modules + 1) + "/path_" + str(j + 1)):
for i, section in enumerate(path):
if i == 0:
if j != 0:
path_outputs[j - 1] = cur_input
cur_input = the_input
cur_conv_output = tf.layers.conv2d(
inputs=cur_input,
filters=section[0],
kernel_size=section[1],
padding='valid' if isinstance(section, tuple) else 'same',
dilation_rate = 1 if len(section) < 3 else section[-1],
use_bias=False,
kernel_initializer=kernel_initializer(),
kernel_regularizer=weight_regularizer(),
trainable=make_trainable,
data_format="channels_last" if data_format == "NHWC" else "channels_first",
name="layer_" + str(i + 1))
cur_batch_normalized = tf.layers.batch_normalization(cur_conv_output,
axis=-1 if data_format == "NHWC" else 1,
scale=False,
training=(mode == tf.estimator.ModeKeys.TRAIN),
trainable=make_trainable,
fused=True)
cur_input = tf.nn.relu(cur_batch_normalized)
to_summarize.append(cur_input)
path_outputs[-1] = cur_input
list(layers.summarize_activation(layer) for layer in to_summarize)
with tf.variable_scope("module_" + str(num_previously_built_inception_modules + 1)):
if len(path_outputs) == 1:
return path_outputs[0]
return tf.concat([temp_input for temp_input in path_outputs], -1 if data_format == "NHWC" else 1)
def build_conv_layers_for_input(self, inputs, name, previous_tensors=None):
"""build_conv_layers_for_input
Creates 2 convolutional layers based on an input.
Changeable parts here are:
Number of outputs for both layers
Size of the kernel used
The stride used
The activation function
:param inputs: The inputs to run the convolutional layers against.
:param name: The name of the input, to scope the layers.
"""
conv_layer1 = layers.conv2d(
inputs=inputs,
data_format="NHWC",
num_outputs=16,
kernel_size=5,
stride=1,
padding="SAME",
activation_fn=tf.nn.relu,
scope=f"{name}/conv_layer1/model_{self.curriculum_number}",
trainable=self.trainable,
)
conv_layer2 = layers.conv2d(
inputs=conv_layer1,
data_format="NHWC",
num_outputs=32,
kernel_size=3,
stride=1,
padding="SAME",
activation_fn=None,
scope=f"{name}/conv_layer2/model_{self.curriculum_number}",
trainable=self.trainable,
)
if self.trainable:
layers.summarize_activation(conv_layer1)
layers.summarize_activation(conv_layer2)
tf.summary.image(
f"{name}/new_conv_layer1", tf.reshape(conv_layer1, [-1, 32, 32, 1]), 3
)
tf.summary.image(
f"{name}/new_conv_layer2", tf.reshape(conv_layer2, [-1, 32, 32, 1]), 3
)
# If we aren't doing transfer learning, return now.
if previous_tensors is None:
return conv_layer2
# Sort the previous models
previous_conv_layer2 = []
for model_number, prev_out in enumerate(previous_tensors):
conv_layer2_previous = layers.conv2d(
inputs=prev_out,
data_format="NHWC",
num_outputs=32,
kernel_size=3,
stride=1,
padding="SAME",
activation_fn=None,
scope=f"{name}/conv_layer2/model_{model_number}",
trainable=self.trainable,
)
previous_conv_layer2.append(conv_layer2_previous)
previous_conv_layer2_added = self.add_all_previous(
previous_conv_layer2, f"{name}/conv_layer2"
)
combined_conv_layer2 = tf.add(
conv_layer2, previous_conv_layer2_added, "%s_conv_add" % name
)
relu_conv_layer2 = tf.nn.relu(
combined_conv_layer2, name="combined_%s_conv_layer2_relu" % name
)
if self.trainable:
layers.summarize_activation(relu_conv_layer2)
tf.summary.image(
f"{name}/combined_conv_layer2",
tf.reshape(relu_conv_layer2, [-1, 32, 32, 1]),
3,
)
return relu_conv_layer2
