def build_cnn_model(n_features: int, n_classes: int):
"""Build the P1FP(C) model using Keras."""
model = keras.Sequential()
model.add(layers.Reshape((1, n_features, 1), input_shape=(n_features, ),
name="input"))
model.add(layers.Conv2D(
128, 12, activation="relu", kernel_regularizer="l2", padding="same"))
model.add(layers.MaxPool2D(10, padding="same"))
model.add(layers.Lambda(nn.local_response_normalization))
model.add(layers.Conv2D(
128, 12, activation="relu", kernel_regularizer="l2", padding="same"))
model.add(layers.MaxPool2D(10, padding="same"))
model.add(layers.Lambda(nn.local_response_normalization))
# It is flattened for the computation regardless, however tflearn retained
# the flattened result whereas keras does not
model.add(layers.Flatten())
model.add(layers.Dense(256, activation="tanh"))
model.add(layers.Dropout(rate=0.2))
model.add(layers.Dense(n_classes, activation="softmax", name="target"))
learning_rate = keras.optimizers.schedules.ExponentialDecay(
0.05, decay_steps=1000, decay_rate=0.96)
model.compile(
optimizer=keras.optimizers.SGD(learning_rate=learning_rate),
loss="categorical_crossentropy",
metrics=[keras.metrics.TopKCategoricalAccuracy(3), "accuracy"])
return model
def build_two_branch_model(inputs):
"""Two parallel convolutions followed by Add and Concat."""
branch1 = layers.Conv2D(filters=10, kernel_size=3, name='conv1')(inputs)
branch1 = layers.BatchNormalization(name='bn1')(branch1)
branch1 = layers.Activation(tf.nn.relu, name='activation1')(branch1)
branch2 = layers.Conv2D(filters=10, kernel_size=3, name='conv2')(inputs)
branch2 = layers.BatchNormalization(name='bn2')(branch2)
branch2 = layers.Activation(tf.nn.relu, name='activation2')(branch2)
merge = layers.Add()([branch1, branch2])
concat = layers.Concatenate(axis=-1)([merge, branch1, branch2])
return concat, branch1, branch2
def testConfigurableConv2DFunctionality(self):
out = ops.ConfigurableConv2D(filters=5, kernel_size=2)(self.inputs)
expected = keras_layers.Conv2D(filters=5, kernel_size=2)(self.inputs)
self.assertAllEqual(out.shape, expected.shape)
self.assertIn(
'configurable_conv2d/ConfigurableConv2D',
[op.name for op in tf.get_default_graph().get_operations()])
def build_simple_keras_model(inputs):
"""Builds lightweight Keras model."""
x = inputs
x = layers.Conv2D(filters=10, kernel_size=3, name='conv')(x)
x = layers.BatchNormalization(name='bn')(x)
x = layers.Activation(tf.nn.relu, name='activation')(x)
return x
def get_keras_layers_for_mnist_experiment(num_components):
"""Get Keras layers for the MNIST experiment.
Args:
num_components: (int) number of components to use for every layer.
Returns:
A list of lists of `keras.layer.Layer`s, where the outer index corresponds
to layer id, and inner index to component id within a layer.
"""
keras_layers = []
filters = 4
keras_layers.append([
layers.Conv2D(filters=filters, kernel_size=5, activation="relu")
for _ in range(num_components)
])
keras_layers.append([layers.AveragePooling2D(pool_size=2)])
keras_layers.append([
layers.Conv2D(filters=filters, kernel_size=3, activation="relu")
for _ in range(num_components)
])
keras_layers.append([layers.AveragePooling2D(pool_size=2)])
keras_layers.append([
layers.Conv2D(filters=filters, kernel_size=3, activation="relu")
for _ in range(num_components)
])
keras_layers.append([layers.Flatten()])
keras_layers.append([layers.Dropout(0.5)])
return keras_layers
def identity_block_base(input_tensor,
kernel_size,
filters,
stage,
block,
num_updates,
dropout_rate=0.,
use_variational_layers=False):
"""The identity block is the block that has no conv layer at shortcut.
Arguments:
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
num_updates: integer, total steps in an epoch (for weighting the loss)
dropout_rate: float, always-on dropout rate.
use_variational_layers: boolean, if true train a variational model
Returns:
x: Output tensor for the block.
"""
filters1, filters2, filters3 = filters
divergence_fn = lambda q, p, ignore: (tfd.kl_divergence(q, p) / num_updates
)
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
if not use_variational_layers:
first_conv_2d = layers.Conv2D(
filters1, (1, 1),
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2a')
if dropout_rate > 0.:
x = layers.Dropout(dropout_rate)(input_tensor, training=True)
x = first_conv_2d(x)
else:
x = first_conv_2d(input_tensor)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2a')(x)
x = layers.Activation('relu')(x)
if dropout_rate > 0.:
x = layers.Dropout(dropout_rate)(x, training=True)
x = layers.Conv2D(filters2,
kernel_size,
use_bias=False,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2b')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2b')(x)
x = layers.Activation('relu')(x)
if dropout_rate > 0.:
x = layers.Dropout(dropout_rate)(x, training=True)
x = layers.Conv2D(filters3, (1, 1),
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2c')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2c')(x)
else:
x = tfpl.Convolution2DFlipout(
filters1,
kernel_size=(1, 1),
padding='SAME',
name=conv_name_base + '2a',
kernel_divergence_fn=divergence_fn,
)(input_tensor)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2a')(x)
x = layers.Activation('relu')(x)
x = tfpl.Convolution2DFlipout(
filters2,
kernel_size=kernel_size,
padding='SAME',
activation=None,
name=conv_name_base + '2b',
kernel_divergence_fn=divergence_fn,
)(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2b')(x)
x = layers.Activation('relu')(x)
x = tfpl.Convolution2DFlipout(
filters3,
kernel_size=(1, 1),
padding='SAME',
activation=None,
name=conv_name_base + '2c',
kernel_divergence_fn=divergence_fn,
)(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = layers.Activation('relu')(x)
return x
def ResNet50(method, num_classes, num_updates, dropout_rate):
"""Instantiates the ResNet50 architecture.
Args:
method: `str`, method for accounting for uncertainty. Must be one of
['vanilla', 'll_dropout', 'll_svi', 'dropout', 'svi', 'dropout_nofirst']
num_classes: `int` number of classes for image classification.
num_updates: integer, total steps in an epoch (for weighting the loss)
dropout_rate: Dropout rate for ll_dropout, dropout methods.
Returns:
A Keras model instance.
pylint: disable=invalid-name
"""
# Determine proper input shape
if backend.image_data_format() == 'channels_first':
input_shape = (3, 224, 224)
bn_axis = 1
else:
input_shape = (224, 224, 3)
bn_axis = 3
if (method in ['dropout', 'll_dropout', 'dropout_nofirst'
]) != (dropout_rate > 0.):
raise ValueError(
'Dropout rate should be nonzero iff a dropout method is used.'
'Method is {}, dropout is {}.'.format(method, dropout_rate))
use_variational_layers = method == 'svi'
hidden_layer_dropout = dropout_rate if method in [
'dropout', 'dropout_nofirst'
] else 0.
img_input = layers.Input(shape=input_shape)
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(img_input)
if (dropout_rate > 0.) and (method != 'dropout_nofirst'):
x = layers.Dropout(hidden_layer_dropout)(x, training=True)
x = layers.Conv2D(64, (7, 7),
use_bias=False,
strides=(2, 2),
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name='bn_conv1')(x)
x = layers.Activation('relu')(x)
x = layers.ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
conv_block = functools.partial(
conv_block_base,
num_updates=num_updates,
dropout_rate=hidden_layer_dropout,
use_variational_layers=use_variational_layers)
identity_block = functools.partial(
identity_block_base,
num_updates=num_updates,
dropout_rate=hidden_layer_dropout,
use_variational_layers=use_variational_layers)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
if dropout_rate > 0.:
x = layers.Dropout(dropout_rate)(x, training=True)
if method in ['ll_svi', 'svi']:
x = tfpl.dense_variational_v2.DenseVariational(
units=num_classes,
make_posterior_fn=posterior_mean_field,
make_prior_fn=functools.partial(prior_trainable,
num_updates=num_updates),
use_bias=True,
kl_weight=1. / num_updates,
kl_use_exact=True,
name='fc1000')(x)
else:
x = layers.Dense(num_classes,
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='fc1000')(x)
# Create model.
return models.Model(img_input, x, name='resnet50')
def get_components_layer_for_general_diversity_and_depth_model(
layer_description, num_filters, group_norm_num_groups, layer_strides):
"""Create a single routed layer.
Args:
layer_description: (list of string) description of the layer. Each element
of the list described one component as a pipe-separated sequence of
operations. Each operation should be one of the following: `convAxB`,
`maxpoolAxB`, `avgpoolAxB`, `identity` where `A` and `B` are integers.
num_filters: (int) number of filters for each convolution.
group_norm_num_groups: (int) number of groups to use for group
normalization; has to divide `num_filters`.
layer_strides: (int) strides for this routed layer. In order to stride
a routed layer, we stride every component within a layer. Since some
components contain several operations (e.g. two convolutions), we apply
the stride to the last operation in each component. To these operations
we pass the value of `layer_strides` as the `strides` argument.
Returns:
A list of `keras.layer.Layer`s, corresponding to components in the routed
layer.
"""
components = []
for component_description in layer_description:
element_descriptions = component_description.split("|")
component_elements = []
for i, element_description in enumerate(element_descriptions):
if i == len(element_descriptions) - 1:
strides = layer_strides
else:
strides = 1
if element_description == "identity":
if strides == 1:
component_elements.append(layers.Lambda(lambda x: x))
else:
# In case of strides, apply an additional 1x1 convolution.
component_elements.append(
layers.Conv2D(filters=num_filters,
kernel_size=1,
padding="same",
strides=strides))
elif element_description.startswith("conv"):
kernel_size = parse_kernel_size(
element_description[len("conv"):])
component_elements.append(
layers.Conv2D(filters=num_filters,
kernel_size=kernel_size,
padding="same",
strides=strides))
component_elements.append(
GroupNorm(num_groups=group_norm_num_groups))
component_elements.append(layers.ReLU())
elif element_description.startswith("maxpool"):
pool_size = parse_kernel_size(
element_description[len("maxpool"):])
component_elements.append(
layers.MaxPooling2D(pool_size=pool_size,
padding="same",
strides=strides))
elif element_description.startswith("avgpool"):
pool_size = parse_kernel_size(
element_description[len("avgpool"):])
component_elements.append(
layers.AveragePooling2D(pool_size=pool_size,
padding="same",
strides=strides))
else:
assert False
components.append(models.Sequential(component_elements))
return components
def get_keras_layers_for_general_diversity_and_depth_model(
layer_description, num_filters, num_layers, num_downsamples,
group_norm_num_groups):
"""Gets Keras layers for the Omniglot and CIFAR-100 experiments.
This model is a generalized version of the one proposed by the authors of
"Diversity and Depth in Per-Example Routing Models"
(https://openreview.net/pdf?id=BkxWJnC9tX).
Args:
layer_description: (list of string) description of a single layer, see
`get_components_layer_for_general_diversity_and_depth_model`.
num_filters: (int) number of filters for each convolution.
num_layers: (int) number of layers.
num_downsamples: (int) number of times the input should be downsampled by a
factor of 2 before reaching the linear task-specific heads.
group_norm_num_groups: (int) number of groups to use for group
normalization.
Returns:
A list of lists of `keras.layer.Layer`s, where the outer index corresponds
to layer id, and inner index to component id within a layer.
"""
keras_layers = []
# Initial shared 1x1 convolution, which increases the number of channels
# from 1 to `num_filters`.
keras_layers.append(
[layers.Conv2D(filters=num_filters, kernel_size=1, padding="same")])
keras_layers.append([GroupNorm(num_groups=group_norm_num_groups)])
keras_layers.append([layers.ReLU()])
downsampling_interval = num_layers / num_downsamples
# Subset of `range(0, num_layers)` - subset of layers for downsampling.
downsampling_layers = [
int(downsampling_interval * i) for i in range(num_downsamples)
]
for layer_id in range(num_layers):
if layer_id in downsampling_layers:
layer_strides = 2
else:
layer_strides = 1
keras_layers.append(
get_components_layer_for_general_diversity_and_depth_model(
layer_description, num_filters, group_norm_num_groups,
layer_strides))
keras_layers.append([GroupNorm(num_groups=group_norm_num_groups)])
keras_layers.append([layers.ReLU()])
# At this point, the feature map is `2^num_downsamples` times smaller.
keras_layers.append([layers.Flatten()])
keras_layers.append([layers.Dropout(0.5)])
return keras_layers
def basic_conv_lstm_cell(inputs,
state,
num_channels,
kernel_size=5,
strides=1,
forget_bias=1.0):
"""Basic LSTM recurrent network cell, with 2D convolution connctions.
We add forget_bias (default: 1) to the biases of the forget gate in order to
reduce the scale of forgetting in the beginning of the training.
It does not allow cell clipping, a projection layer, and does not
use peep-hole connections: it is the basic baseline.
Args:
inputs: input Tensor, 4D, batch x height x width x channels.
state: state Tensor, 4D, batch x height x width x channels.
num_channels: the number of output channels in the layer.
kernel_size: the shape of the each convolution filter.
forget_bias: the initial value of the forget biases.
scope: Optional scope for variable_scope.
reuse: whether or not the layer and the variables should be reused.
Returns:
a tuple of tensors representing output and the new state.
"""
def init_state(inputs,
state_shape,
state_initializer=tf.zeros_initializer(),
dtype=tf.float32):
"""Helper function to create an initial state given inputs.
Args:
inputs: input Tensor, at least 2D, the first dimension being batch_size
state_shape: the shape of the state.
state_initializer: Initializer(shape, dtype) for state Tensor.
dtype: Optional dtype, needed when inputs is None.
Returns:
A tensors representing the initial state.
"""
# Handle both the dynamic shape as well as the inferred shape.
inferred_batch_size = inputs.get_shape().as_list()[0]
dtype = inputs.dtype
initial_state = state_initializer([inferred_batch_size] + state_shape, dtype=dtype)
return initial_state
spatial_size = inputs.get_shape().as_list()[1:3]
if state is None:
state = init_state(inputs, spatial_size + [2 * num_channels]) # (stack c, h)
with v1.variable_scope('BasicConvLstmCell'):
inputs.get_shape().assert_has_rank(4)
state.get_shape().assert_has_rank(4)
c, h = tf.split(axis=3, num_or_size_splits=2, value=state)
inputs_h = tf.concat(axis=3, values=[inputs, h])
# Parameters of gates are concatenated into one conv for efficiency.
i_j_f_o = Layers.Conv2D(4 * num_channels, kernel_size=(kernel_size, kernel_size), \
strides=(strides, strides), padding="same")(inputs_h)
# i = input_gate, j = new_input, f = forget_gate, o = output_gate
i, j, f, o = tf.split(axis=3, num_or_size_splits=4, value=i_j_f_o)
new_c = c * tf.sigmoid(f + forget_bias) + tf.sigmoid(i) * tf.tanh(j)
new_h = tf.tanh(new_c) * tf.sigmoid(o)
return new_h, tf.concat(axis=3, values=[new_c, new_h])
def stack_layers(inputs, net_layers, kernel_initializer='glorot_uniform'):
"""Builds the architecture of the network by applying each layer specified in net_layers to inputs.
Args:
inputs: a dict containing input_types and input_placeholders for each key
and value pair, respecively.
net_layers: a list of dicts containing all layers to be used in the
network, where each dict describes one such layer. each dict requires the
key 'type'. all other keys are dependent on the layer type.
kernel_initializer: initialization configuration passed to keras (see keras
initializers).
Returns:
outputs: a dict formatted in much the same way as inputs. it
contains input_types and output_tensors for each key and value pair,
respectively, where output_tensors are the outputs of the
input_placeholders in inputs after each layer in net_layers is applied.
"""
outputs = dict()
for key in inputs:
outputs[key] = inputs[key]
for layer in net_layers:
# check for l2_reg argument
l2_reg = layer.get('l2_reg')
if l2_reg:
l2_reg = l2(layer['l2_reg'])
# create the layer
if layer['type'] in [
'softplus', 'softsign', 'softmax', 'tanh', 'sigmoid', 'relu', 'selu'
]:
l = layers.Dense(
layer['size'],
activation=layer['type'],
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'None':
l = layers.Dense(
layer['size'],
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'Conv2D':
l = layers.Conv2D(
layer['channels'],
kernel_size=layer['kernel'],
activation='relu',
data_format='channels_last',
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'BatchNormalization':
l = layers.BatchNormalization(name=layer.get('name'))
elif layer['type'] == 'MaxPooling2D':
l = layers.MaxPooling2D(
pool_size=layer['pool_size'],
data_format='channels_first',
name=layer.get('name'))
elif layer['type'] == 'Dropout':
l = layers.Dropout(layer['rate'], name=layer.get('name'))
elif layer['type'] == 'Flatten':
l = layers.Flatten(name=layer.get('name'))
else:
raise ValueError("Invalid layer type '{}'".format(layer['type']))
# apply the layer to each input in inputs
for k in outputs:
outputs[k] = l(outputs[k])
return outputs
def call(self, inputs, mode="train"):
"""
Forward function for pixel advection network
Parameters:
inputs: input dictionary including "image", "r_state" and "action"
mode: specify training or validating/testing
Return:
gen_images: list of generated images
gen_states: list of generated states
"""
##### preparations #####
# get dimensions/global steps
global_step = tf.cast(v1.train.get_or_create_global_step(), tf.float32)
isTrain = True if mode == "train" else False
batch_size, image_height, image_width, color_ch = inputs["image"][
0].get_shape().as_list()[0:4]
state_dim = inputs["r_state"][0].get_shape().as_list()[1]
# placeholder for generated robot states and images
gen_states, gen_images = [], []
# initial r state will use ground truth
current_r_state = inputs["r_state"][0]
# placeholder for conv-lstm states
lstm_state1, lstm_state2, lstm_state3, lstm_state4 = None, None, None, None
lstm_state5, lstm_state6, lstm_state7 = None, None, None
# get number of ground truth images used for each mini-batch
num_ground_truth = tf.cast(
tf.round(
tf.cast(batch_size, tf.float32) *
(self.k / (self.k + tf.exp(global_step / self.k)))), tf.int32)
###### begin time-step loop (total_len - 1 steps) ######
for image, action in zip(inputs["image"][:-1], inputs["action"][:-1]):
##### sampling and updating values #####
# reuse parameters after the first step
reuse = None if not bool(gen_images) else True
# warm start(use ground truth frames) in first context_len steps
done_warm_start = len(gen_images) > self.context_len - 1
# if using context frames (during warm start), always use ground truth input
# else, if not explicitly specified by "use_predict_frame", choose to use generated image
# or ground truth input based on sampling function
if self.use_predict_frame and done_warm_start:
current_image = gen_images[-1]
elif done_warm_start:
current_image = self.scheduled_sample(image, gen_images[-1],
batch_size,
num_ground_truth)
else:
current_image = image
# concat state and action, always use ground truth action, but use current state
current_state_action = tf.concat([action, current_r_state], axis=1)
##### begin U-net #####
# 1th conv
with v1.variable_scope("conv1", reuse=reuse):
enc0 = Layers.Conv2D(self.layer_ch_specs[0],
kernel_size=(5, 5),
strides=(2, 2),
padding="same")(current_image)
enc0 = Layers.LayerNormalization()(enc0)
# 1th conv lstm
with v1.variable_scope("conv_lstm1", reuse=reuse):
hidden1, lstm_state1 = basic_conv_lstm_cell(
enc0, lstm_state1, self.layer_ch_specs[1])
hidden1 = Layers.LayerNormalization()(hidden1)
# 2th conv lstm
with v1.variable_scope("conv_lstm2", reuse=reuse):
hidden2, lstm_state2 = basic_conv_lstm_cell(
hidden1, lstm_state2, self.layer_ch_specs[2])
hidden2 = Layers.LayerNormalization()(hidden2)
enc1 = Layers.Conv2D(self.layer_ch_specs[2], kernel_size=(3, 3), strides=(2, 2), padding="same")\
(hidden2)
# 3th conv lstm
with v1.variable_scope("conv_lstm3", reuse=reuse):
hidden3, lstm_state3 = basic_conv_lstm_cell(
enc1, lstm_state3, self.layer_ch_specs[3])
hidden3 = Layers.LayerNormalization()(hidden3)
# 4th conv lstm
with v1.variable_scope("conv_lstm4", reuse=reuse):
hidden4, lstm_state4 = basic_conv_lstm_cell(
hidden3, lstm_state4, self.layer_ch_specs[4])
hidden4 = Layers.LayerNormalization()(hidden4)
enc2 = Layers.Conv2D(self.layer_ch_specs[4], kernel_size=(3, 3), strides=(2, 2), padding="same")\
(hidden4)
# Pass in state and action.
smear = tf.reshape(current_state_action,
[batch_size, 1, 1, state_dim * 2])
smear = tf.tile(
smear,
[1,
int(enc2.get_shape()[1]),
int(enc2.get_shape()[2]), 1])
if self.use_state:
enc2 = tf.concat(axis=3, values=[enc2, smear])
enc3 = Layers.Conv2D(self.layer_ch_specs[4], kernel_size=(1, 1), strides=(1, 1), padding="same")\
(enc2)
# 5th conv lstm
with v1.variable_scope("conv_lstm5", reuse=reuse):
hidden5, lstm_state5 = basic_conv_lstm_cell(
enc3, lstm_state5, self.layer_ch_specs[5])
hidden5 = Layers.LayerNormalization()(hidden5)
enc4 = Layers.Conv2DTranspose(self.layer_ch_specs[5], kernel_size=(3, 3), strides=(2, 2), padding="same")\
(hidden5)
# 6th conv lstm
with v1.variable_scope("conv_lstm6", reuse=reuse):
hidden6, lstm_state6 = basic_conv_lstm_cell(
enc4, lstm_state6, self.layer_ch_specs[6])
hidden6 = Layers.LayerNormalization()(hidden6)
# Skip connection.
hidden6 = tf.concat(axis=3, values=[hidden6,
enc1]) # both 16x16
enc5 = Layers.Conv2DTranspose(self.layer_ch_specs[6], kernel_size=(3, 3), strides=(2, 2), padding="same")\
(hidden6)
# 7th conv lstm
with v1.variable_scope("conv_lstm7", reuse=reuse):
hidden7, lstm_state7 = basic_conv_lstm_cell(
enc5, lstm_state7, self.layer_ch_specs[7]) # 32x32
hidden7 = Layers.LayerNormalization()(hidden7)
# Skip connection.
hidden7 = tf.concat(axis=3, values=[hidden7,
enc0]) # both 32x32
enc6 = Layers.Conv2DTranspose(self.layer_ch_specs[7], kernel_size=(3, 3), strides=(2, 2), padding="same")\
(hidden7)
enc6 = Layers.LayerNormalization()(enc6)
###### motion transform part #####
# dna
if self.dna:
from video_prediction.models.building_blocks import dna_transformation
with v1.variable_scope("dna", reuse=reuse):
if self.num_mask != 1:
raise ValueError(
'Only one mask is supported for DNA model.')
dna_input = Layers.Conv2DTranspose(self.dna_kernel_size ** 2, kernel_size=(1, 1), strides=(1, 1),\
padding="same")(enc6)
transformed = [dna_transformation(current_image, dna_input, dna_kernel_size=self.dna_kernel_size, \
relu_shift=self.relu_shift)]
# cdna
elif self.cdna:
from video_prediction.models.building_blocks import cdna_transformation
with v1.variable_scope("cdna", reuse=reuse):
last_hidden_input = Layers.Conv2DTranspose(color_ch, kernel_size=(1, 1), strides=(1, 1),\
padding="same")(enc6)
transformed = [
keras.activations.sigmoid(last_hidden_input)
]
cdna_input = tf.reshape(hidden5, [batch_size, -1])
transformed += cdna_transformation(current_image, cdna_input, num_masks=self.num_mask, \
color_channels=color_ch, dna_kernel_size=self.dna_kernel_size, relu_shift=self.relu_shift)
# stp
elif self.stp:
assert (0)
from video_prediction.models.building_blocks import stp_transformation
with v1.variable_scope("stp", reuse=reuse):
last_hidden_input = Layers.Conv2DTranspose(color_ch, kernel_size=(1, 1), strides=(1, 1),\
padding="same")(enc6)
transformed = [
keras.activations.sigmoid(last_hidden_input)
]
stp_input = tf.reshape(hidden5, [batch_size, -1])
stp_input = Layers.Dense(100)(stp_input)
transformed += stp_transformation(current_image, stp_input,
self.num_mask)
# compute mask
with v1.variable_scope("mask", reuse=reuse):
mask = Layers.Conv2DTranspose(self.num_mask + 1, kernel_size=(1, 1), strides=(1, 1), padding="same")\
(enc6)
mask = tf.reshape(
tf.nn.softmax(tf.reshape(mask, [-1, self.num_mask + 1])),
[batch_size, image_height, image_width, self.num_mask + 1])
#layers.append(("softmax_mask", mask))
mask_list = tf.split(axis=3,
num_or_size_splits=self.num_mask + 1,
value=mask)
# mask output
# first mask applies to current_image
new_gen_image = mask_list[0] * current_image
for layer, mask in zip(transformed, mask_list[1:]):
new_gen_image += layer * mask
gen_images.append(new_gen_image)
###### compute new r state #####
new_gen_r_state = Layers.Dense(state_dim)(current_state_action)
gen_states.append(new_gen_r_state)
# update current state
current_r_state = new_gen_r_state
return gen_images, gen_states
ytest) = tf.compat.v1.keras.datasets.cifar100.load_data(
label_mode='fine')
xtrain = xtrain / 255
xtest = xtest / 255
#training = tf.compat.v2.data.Dataset.from_tensor_slices((xtrain,ytrain))
#testing = tf.compat.v2.data.Dataset.from_tensor_slices((xtest,ytest))
#training = training.batch(64).shuffle(buffer_size=64)
#testing = testing.batch(64).shuffle(buffer_size=64)
#training = training.prefetch(tf.data.experimental.AUTOTUNE)
#testing = testing.prefetch(tf.data.experimental.AUTOTUNE)
model = tf.compat.v1.keras.Sequential()
model.add(
layers.Conv2D(filters=64,
kernel_size=4,
strides=2,
padding='valid',
use_bias=True,
input_shape=(32, 32, 3)))
model.add(layers.BatchNormalization())
model.add(layers.Activation(tf.nn.leaky_relu))
model.add(layers.Conv2D(128, 4, 2, 'valid', use_bias=True))
model.add(layers.BatchNormalization())
model.add(layers.Activation(tf.nn.leaky_relu))
model.add(layers.Conv2D(256, 1, 1, 'valid', use_bias=True))
model.add(layers.BatchNormalization())
model.add(layers.Conv2D(256, 1, 1, 'valid', use_bias=True))
model.add(layers.Flatten())
model.add(layers.Dense(100, activation='softmax'))
model.compile(optimizer='Adam',
loss=tf.compat.v1.keras.losses.SparseCategoricalCrossentropy(),
test_data = dataset[int(dataset.shape[0] * 0.75):]
test_data = test_data.reshape(85794, 20, 7)
test_data = np.expand_dims(test_data, axis=3)
print(test_data.shape)
test_data_labels = keras.utils.to_categorical(
dataset_labels[int(dataset.shape[0] * 0.75):])
# Init statemnts
print(tf.__version__)
# Define model
model = models.Sequential()
model.add(layers.InputLayer(input_shape=(20, 7, 1)))
model.add(layers.Conv2D(32, (3, 2), activation='relu'))
model.add(layers.MaxPooling2D((2, 1)))
model.add(layers.Conv2D(64, (2, 1), activation='relu'))
model.add(layers.MaxPooling2D((2, 1)))
model.add(layers.Conv2D(128, (2, 1), activation='relu'))
model.add(layers.Flatten())
# model.add(layers.Dense(128, activation='relu'))
# model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(12, activation='softmax'))
# Init TensorBoard callback for data visualization in real time
tensorboard = TensorBoard(log_dir="logs/test".format(time()))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=[
