def generate_cnn():
"""
Function to generate a Convolutional Neural Network
to estimate entropy from (56,56,56) voxel occupancy grids.
:return: Keras.Model
"""
inputs = keras.Input(shape=(56, 56, 56))
base = layers.Reshape(target_shape=(56, 56, 56, 1))(inputs)
# cnn_a_filters = hp.Int('cnn1_filters', min_value=4, max_value=16, step=4)
a = layers.Conv3D(8, (5, 5, 5), activation='relu', padding='same')(base)
a = layers.AveragePooling3D(pool_size=(2, 2, 2))(a)
a = layers.BatchNormalization()(a)
a = layers.Dropout(0.25)(a)
a = layers.Flatten()(a)
# cnn_b_filters = hp.Int('cnn2_filters', min_value=4, max_value=16, step=4)
b = layers.Conv3D(8, (3, 3, 3), activation='relu', padding='same')(base)
b = layers.AveragePooling3D(pool_size=(2, 2, 2))(b)
b = layers.BatchNormalization()(b)
b = layers.Dropout(0.25)(b)
b = layers.Flatten()(b)
x = layers.Concatenate(axis=1)([a, b])
# dense_units = hp.Int('dense_units', min_value=256, max_value=512, step=64)
x = layers.Dense(512, activation='relu')(x)
x = layers.BatchNormalization()(x)
x = layers.Dropout(0.5)(x)
outputs = layers.Dense(60, activation='linear')(x)
model = keras.Model(inputs=inputs, outputs=outputs, name='entronet')
model.compile(optimizer=keras.optimizers.Adam(learning_rate=5e-5), loss='mae', metrics=['mse'])
model.summary()
return model
def __build(self, is_training):
#define inputs:[batch_size, Depth, Height, Width, Channels], for keras, you don't need
#to specify the batch_size
dtype = tf.float32
input_image = KL.Input(shape = self.input_shape + [1], dtype= dtype, name='input_image')
base_nb_filters = 16
x = KL.Conv3D(base_nb_filters, (5, 5, 5), (1, 1, 1), padding='same')(input_image)
x = KL.BatchNormalization(axis=-1)(x, training=is_training)
x = KL.ReLU()(x)
for k in range(self.n_groups):
nb_filters = base_nb_filters * 2**(k)
for l in range(self.n_blocks_per_group[k]):
x = ResnetBlock_SelfAtten_3D(x, nb_filters, training=is_training)
nb_filters = base_nb_filters * 2**(k+1)
x = DownSampling_Unilateral_3D(x, nb_filters, training= is_training)
x = KL.AveragePooling3D()(x)
flatten = KL.Flatten(name='flatten')(x)
fc1 = KL.Dense(512, activation='relu', name='fc1')(flatten)
fc2 = KL.Dense(1024, activation='relu', name='fc2')(fc1)
y_preds = KL.Dense(self.num_classes, activation='softmax', name='y_preds')(fc2)
model = KM.Model(input_image, y_preds, name=self.NET_NAME.lower())
return model
def pooling_unit(conv1_1, stage, pooling='max', kernel_size=2, stride=2):
if pooling=='max':
return layers.MaxPooling3D((2, 2, 2), strides=(2, 2, 2), name='pool'+str(stage))(conv1_1)
elif pooling=='avg':
return layers.AveragePooling3D((2, 2, 2), strides=(2, 2, 2), name='pool'+str(stage))(conv1_1)
else:
return conv1_1
def TemporalTransitionLayer(inputs, TTL_config, i):
x1 = layers.BatchNormalization()(inputs)
x1 = layers.Activation('relu')(x1)
x1 = layers.Conv3D(filters=128,
kernel_size=(1,1,1),
activation=None,
padding='same',
use_bias=False)(x1)
x2 = layers.BatchNormalization()(inputs)
x2 = layers.Activation('relu')(x2)
x2 = layers.Conv3D(filters=128,
kernel_size=(3,3,3),
activation=None,
padding='same',
use_bias=False)(x2)
x3 = layers.BatchNormalization()(inputs)
x3 = layers.Activation('relu')(x3)
x3 = layers.Conv3D(filters=128,
kernel_size=(TTL_config[i],3,3),
activation=None,
padding='same',
use_bias=False)(x3)
x = tf.concat([x1, x2, x3], 4)
x = layers.AveragePooling3D(pool_size=(2,2,2),
strides=(2,2,2))(x)
return(x)
def __init__(self, kernel: tuple, stride=2, padding=0, scope='APOOL'):
super(AvgPool, self).__init__(scope)
dim = len(kernel)
is_global = True if kernel[-1]<0 else False
if is_global:
assert padding == 0
if dim == 1:
if is_global:
self.pool = _AP(1)
else:
self.pool = layers.AveragePooling1D(kernel, stride)
pad_fn = layers.ZeroPadding1D
elif dim == 2:
if is_global:
self.pool = _AP(2)
else:
self.pool = layers.AveragePooling2D(kernel, stride)
pad_fn = layers.ZeroPadding2D
elif dim == 3:
if is_global:
self.pool = _AP(3)
else:
self.pool = layers.AveragePooling3D(kernel, stride)
pad_fn = layers.ZeroPadding3D
else:
raise Exception('NEBULAE ERROR ⨷ %d-d pooling is not supported.' % dim)
if isinstance(padding, int):
padding = dim * [[padding, padding]]
elif isinstance(padding, (list, tuple)):
padding = [(padding[2*d], padding[2*d+1]) for d in range(dim-1, -1, -1)]
self.pad = pad_fn(padding)
def Transition_DenseNet(input):
x = layers.BatchNormalization()(inputs)
x = layers.Activation('relu')(x)
x = layers.Conv3D(filters=bn_size * growth_rate,
kernel_size=1,
activation=None,
padding='same',
use_bias=False)(x)
x = layers.AveragePooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2))
return (x)
def pyramidal_make_layer(x, block, block_depth, stride, featuremap_dim,
addrate):
downsample = None
if stride != 1:
downsample = layers.AveragePooling3D((2, 2, 1), strides=(2, 2, 1))
featuremap_dim = featuremap_dim + addrate
x = block(x, int(round(featuremap_dim)), stride, downsample)
for i in range(1, block_depth):
temp_featuremap_dim = featuremap_dim + addrate
x = block(x, int(round(temp_featuremap_dim)), 1)
featuremap_dim = temp_featuremap_dim
return x, featuremap_dim
def __init__(self):
super(ResC3D, self).__init__()
self.reshape = tf.keras.layers.Reshape((3, 32, 112, 112))
self.conv1 = layers.Conv3D(64, (7, 7, 3),
strides=(2, 1, 1),
padding='same',
activation='relu')
self.conv2a = layers.Conv3D(64, (1, 1, 1),
strides=(1, 1, 1),
padding='same',
activation='relu')
self.conv2 = layers.Conv3D(64, (3, 3, 3),
padding='same',
activation='relu')
self.residual = layers.Add()
self.conv3a = layers.Conv3D(128, (1, 1, 1),
strides=(2, 2, 1),
padding='same',
activation='relu')
self.conv3 = layers.Conv3D(128, (3, 3, 3),
padding='same',
activation='relu')
self.conv4a = layers.Conv3D(256, (1, 1, 1),
strides=(1, 2, 2),
padding='same',
activation='relu')
self.conv4 = layers.Conv3D(256, (3, 3, 3),
padding='same',
activation='relu')
self.conv5a = layers.Conv3D(512, (1, 1, 1),
strides=(1, 2, 2),
padding='same',
activation='relu')
self.conv5 = layers.Conv3D(512, (3, 3, 3),
padding='same',
activation='relu')
self.conv6 = layers.Conv3D(1024, (1, 1, 1),
padding='same',
activation='relu')
self.pool = layers.AveragePooling3D((7, 7, 1), padding='same')
self.flatten = layers.Flatten()
self.dense = layers.Dense(249, activation='softmax')
def mydensenet(blocks_in_dense=2, dense_conv_blocks=2, dense_layers=1, num_dense_connections=256, filters=16,
growth_rate=16, reduction=0.5, **kwargs):
"""
:param blocks_in_dense: how many convolution blocks are in a single size layer
:param dense_conv_blocks: how many dense blocks before a max pooling to occur
:param dense_layers: number of dense layers
:param num_dense_connections:
:param filters:
:param growth_rate:
:param kwargs:
:return:
"""
blocks_in_dense = int(blocks_in_dense)
dense_conv_blocks = int(dense_conv_blocks)
dense_layers = int(dense_layers)
num_dense_connections = int(num_dense_connections)
filters = int(filters)
growth_rate = int(growth_rate)
reduction = float(reduction)
input_shape = (32, 64, 64, 2)
img_input = layers.Input(shape=input_shape)
x = img_input
inputs = (img_input,)
x = layers.Conv3D(filters, (3, 7, 7), strides=2, use_bias=False, name='conv1/conv', padding='Same')(x)
# x = layers.BatchNormalization(axis=-1, epsilon=1.001e-5, name='conv1/bn')(x)
x = GroupNormalization(groups=2, axis=-1, name='conv1/gn')(x)
x = layers.Activation('selu', name='conv1/selu')(x)
for i in range(dense_conv_blocks):
x = dense_block3d(x=x, growth_rate=growth_rate, blocks=blocks_in_dense, name='conv{}'.format(i))
x = transition_block(x=x, reduction=reduction, name='pool{}'.format(i))
# x = layers.BatchNormalization(axis=-1, epsilon=1.001e-5, name='bn')(x)
x = GroupNormalization(groups=2, axis=-1, name='gn')(x)
x = layers.Activation('selu', name='selu')(x)
x = layers.AveragePooling3D(pool_size=(2, 2, 2), name='final_average_pooling')(x)
x = layers.Flatten()(x)
for i in range(dense_layers):
x = layers.Dense(num_dense_connections, activation='selu', kernel_regularizer=regularizers.l2(0.001))(x)
x = layers.Dropout(0.5)(x)
x = layers.Dense(2, activation='softmax', name='prediction', dtype='float32')(x)
model = Model(inputs=inputs, outputs=(x,), name='my_3d_densenet')
return model
def transition_block(x, reduction, name, strides=(2, 2, 2)):
"""A transition block.
Arguments:
x: input tensor.
reduction: float, compression rate at transition layers.
name: string, block label.
Returns:
output tensor for the block.
"""
# x = layers.BatchNormalization(axis=-1, epsilon=1.001e-5, name=name + '_bn')(x)
x = GroupNormalization(groups=2, axis=-1, name=name + '_gn')(x)
x = layers.Activation('selu', name=name + '_selu')(x)
x = layers.Conv3D(int(x.shape[-1] * reduction), 1,
use_bias=False, padding='same', name=name + '_conv')(x)
x = layers.AveragePooling3D(strides, strides=strides, name=name + '_pool')(x)
return x
def train(self, videos, vid_size, start_size, epochs, lr, save_dir, b1, b2, save_int, num_out, **kwargs):
# Load from checkpoint
latest_checkpoint = tf.train.latest_checkpoint(save_dir)
if latest_checkpoint:
print('Loading checkpoint ' + latest_checkpoint)
loop_start_size = start_size * 2**(int(os.path.basename(latest_checkpoint)[4])-1)
else:
loop_start_size = start_size
self.init_models(loop_start_size, lr, b1, b2)
if latest_checkpoint:
self.checkpoint.restore(latest_checkpoint)
# Number of progressive resolution stages
resolutions = int(np.log2(vid_size/loop_start_size)) + 1
for resolution in range(resolutions):
print('Resolution: ', loop_start_size*2**resolution)
self.fade = True if (resolution > 0 or loop_start_size > start_size) else False
for epoch in range(epochs):
start = time.time()
for batch in videos:
if resolution < resolutions - 1:
batch = kl.AveragePooling3D(2**(resolutions - resolution - 1), padding='same')(batch)
fade = epoch/(epochs//2)
if fade == 1:
self.fade = False
self.train_step(videos=tf.cast(batch, tf.float32),
fade=tf.constant(fade, shape=(self.batch_size, 1), dtype=tf.float32))
# Save every n intervals
if (epoch + 1) % save_int == 0:
self.generate(epoch + 1, save_dir, num_out)
if self.save_checkpts:
self.checkpoint.save(file_prefix=os.path.join(save_dir, "ckpt" + str(resolution+1)))
print('Time taken for epoch {} is {} sec'.format(epoch + 1, time.time() - start))
if resolution < resolutions - 1:
print('Updating models to add new layers for next resolution.')
self.update_models(loop_start_size*2**resolution, start_size)
# Generate samples after final epoch
self.generate(epochs, save_dir, num_out)
def discriminator_model(self, out_size, start_filters=512):
# Fading function
def blend_resolutions(upper, lower, alpha):
upper = tf.multiply(upper, alpha)
lower = tf.multiply(lower, tf.subtract(1.0, alpha)[..., tf.newaxis, tf.newaxis, tf.newaxis])
return kl.Add()([upper, lower])
conv_loop = int(np.log2(out_size)) - 3
vid = kl.Input(shape=(out_size/2, out_size, out_size, 3,))
fade = kl.Input(shape=(1,))
# Convert from RGB frames
converted = kl.Conv3D(filters=start_filters, kernel_size=1, strides=1, padding='same',
kernel_initializer=self.conv_init, use_bias=True, name='conv_from_img_'+str(out_size))(vid)
# First convolution downsamples by factor of 2
x = kl.Conv3D(filters=start_filters, kernel_size=4, strides=2, padding='same',
kernel_initializer=self.conv_init, name='conv_'+str(out_size/2))(converted)
# Calculate discriminator score using alpha-blended combination of new discriminator layer outputs
# versus downsampled version of input videos
if self.fade:
downsampled = kl.AveragePooling3D(pool_size=(2, 2, 2), padding='same')(converted)
x = kl.Lambda(lambda args: blend_resolutions(args[0], args[1], args[2]))([x, downsampled, fade])
x = kl.Lambda(lambda x: tf.contrib.layers.layer_norm(x))(x)
x = kl.LeakyReLU(.2)(x)
for resolution in range(conv_loop):
filters = min(out_size * 2 ** (resolution + 1), start_filters)
x = kl.Conv3D(filters=filters, kernel_size=4, strides=2, padding='same',
kernel_initializer=self.conv_init, name='conv_' + str(out_size / 2**(resolution+2)))(x)
x = kl.Lambda(lambda x: tf.contrib.layers.layer_norm(x))(x)
x = kl.LeakyReLU(.2)(x)
# Convert to single value
x = kl.Conv3D(filters=1, kernel_size=4, strides=2, padding='same',
kernel_initializer=self.conv_init, name='conv_1')(x)
x = kl.LeakyReLU(.2)(x)
x = kl.Flatten()(x)
x = kl.Dense(1, kernel_initializer=tf.keras.initializers.random_normal(stddev=0.01),
name='dense_'+str(x.get_shape().as_list()[-1]))(x)
return tf.keras.models.Model(inputs=[vid, fade], outputs=x, name='discriminator')
def define_model_R(nchan, L, Fs, sigmas):
model = tf.keras.Sequential()
model.add(layers.InputLayer((sigmas, Fs, L, nchan), batch_size=1))
model.add(layers.LayerNormalization())
#model.add(layers.BatchNormalization())
#model.add(layers.Conv3D(filters=5, kernel_size=[sigmas,1,1])) # Channels is channels
model.add(layers.Permute((4, 2, 3, 1)))
model.add(
layers.Conv3D(filters=10, kernel_size=[nchan, 1, 1],
activation='elu')) # Freq is channels
model.add(layers.AveragePooling3D(pool_size=(1, 6, 4), strides=(1, 4, 3)))
model.add(layers.Dropout(0.75))
model.add(layers.Flatten())
#model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(3, activation='softmax'))
model.compile(loss=losses.CategoricalCrossentropy(),
optimizer=optimizers.Adam(),
metrics=['accuracy'],
run_eagerly=False)
return model
def f_define_model(inpx, name):
'''
Function that defines the model and compiles it.
'''
inputs = layers.Input(shape=inpx.shape[1:])
h = inputs
# Choose model
if name == '1':
# Convolutional layers
conv_sizes = [6, 6, 6]
conv_args = dict(kernel_size=(3, 3, 3),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.MaxPooling3D(pool_size=(2, 2, 2))(h)
h = layers.Dropout(0.5)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(64, activation='relu')(h)
h = layers.Dropout(0.5)(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '2':
# Convolutional layers
conv_sizes = [20, 20, 20, 20]
conv_args = dict(kernel_size=(2, 2, 2),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.MaxPooling3D(pool_size=(1, 2, 2))(h)
h = layers.Dropout(0.5)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
h = layers.Dropout(0.5)(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '3':
# Convolutional layers
conv_sizes = [40, 40, 40]
conv_args = dict(kernel_size=(2, 4, 12),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.MaxPooling3D(pool_size=(1, 2, 3))(h)
h = layers.Dropout(0.5)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
h = layers.Dropout(0.5)(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '4':
# Convolutional layers
conv_sizes = [40, 40, 40]
conv_args = dict(kernel_size=(2, 4, 12),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.MaxPooling3D(pool_size=(1, 2, 3))(h)
h = layers.Dropout(0.5)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
#h = layers.Dropout(0.5)(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '5':
# Convolutional layers
conv_sizes = [40, 40, 40]
conv_args = dict(kernel_size=(2, 4, 12),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.MaxPooling3D(pool_size=(1, 2, 3))(h)
h = layers.Dropout(0.5)(h)
h = layers.Conv3D(80, **conv_args)(h)
h = layers.Conv3D(120, **conv_args)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
#h = layers.Dropout(0.5)(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
# Testing new models
elif name == '6':
# Convolutional layers
conv_sizes = [40, 40, 40]
conv_args = dict(kernel_size=(2, 4, 12),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.BatchNormalization()(h)
h = layers.MaxPooling3D(pool_size=(1, 2, 3))(h)
h = layers.Dropout(0.5)(h)
h = layers.Conv3D(80, **conv_args)(h)
h = layers.Conv3D(120, **conv_args)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '7':
# Convolutional layers
conv_sizes = [40, 40, 40]
conv_args = dict(kernel_size=(2, 4, 12),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.AveragePooling3D(pool_size=(1, 2, 3))(h)
h = layers.Dropout(0.5)(h)
h = layers.Conv3D(80, **conv_args)(h)
h = layers.Conv3D(120, **conv_args)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '8':
# Convolutional layers
conv_sizes = [40, 40, 40]
conv_args = dict(kernel_size=(2, 4, 12),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.BatchNormalization()(h)
h = layers.AveragePooling3D(pool_size=(1, 2, 3))(h)
h = layers.Dropout(0.5)(h)
h = layers.Conv3D(80, **conv_args)(h)
h = layers.Conv3D(120, **conv_args)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '9':
# Convolutional layers
conv_sizes = [40, 40, 40]
conv_args = dict(kernel_size=(2, 4, 12),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.MaxPooling3D(pool_size=(1, 2, 3))(h)
h = layers.Dropout(0.5)(h)
h = layers.Conv3D(80, **conv_args)(h)
h = layers.Conv3D(120, **conv_args)(h)
h = layers.Conv3D(150, **conv_args)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '10':
# Convolutional layers
conv_sizes = [40, 40, 40]
conv_args = dict(kernel_size=(2, 4, 12),
activation='relu',
padding='same')
for conv_size in conv_sizes:
h = layers.Conv3D(conv_size, **conv_args)(h)
h = layers.BatchNormalization()(h)
h = layers.MaxPooling3D(pool_size=(1, 2, 3))(h)
h = layers.Dropout(0.5)(h)
h = layers.Conv3D(60, **conv_args)(h)
h = layers.Conv3D(80, **conv_args)(h)
h = layers.Conv3D(100, **conv_args)(h)
h = layers.Conv3D(120, **conv_args)(h)
h = layers.Flatten()(h)
# Fully connected layers
h = layers.Dense(120, activation='relu')(h)
# Ouptut layer
outputs = layers.Dense(1, activation='sigmoid')(h)
elif name == '15': # Resnet 50
#from resnet50 import *
model = ResNet50(img_input=inputs)
learn_rate = 0.0005
elif name == '16': # Resnet 50
model = ResNet50(img_input=inputs)
learn_rate = 0.0001
elif name == '17': # Resnet 50
model = ResNet50(img_input=inputs)
learn_rate = 0.00001
elif name == '18': # Resnet 18
model = ResNet18(img_input=inputs)
learn_rate = 0.0005
elif name == '19': # Resnet 18
model = ResNet18(img_input=inputs)
learn_rate = 0.0001
elif name == '20': # Resnet 18
model = ResNet18(img_input=inputs)
learn_rate = 0.00005
elif name == '21': # Resnet 18
model = ResNet18(img_input=inputs)
learn_rate = 0.00001
## Add more models above
############################################
####### Compile model ######################
############################################
if name in ['15', '16', '17', '18', '19', '20', '21']:
opt, loss_fn = optimizers.Adam(
lr=learn_rate), 'sparse_categorical_crossentropy'
else: ## For non resnet models
model = models.Model(inputs, outputs)
learn_rate = 0.00005
#### change loss function for non-resnet models since 'sparse_categorical_crossentropy' throws up an error.
opt, loss_fn = optimizers.Adam(lr=learn_rate), 'binary_crossentropy'
model.compile(optimizer=opt, loss=loss_fn, metrics=['accuracy'])
#print("model %s"%name)
#model.summary()
return model
def _build_custom_model(self):
"""
Builds a flat cnn3d model using tensorflow 2. It has same
number of convolutional kernels throughout.
Args:
params (dict): Dictionary having parameters for architecture.
1. num_conv_layers
2. num_kernels
"""
# Extracting architecture parameters from dictionary
kernels_per_layer_ = self._params["kernels_per_layer"]
kernel_size_ = self._params["kernel_size"]
activation_ = self._params["activation"]
data_format_ = self._params["data_format"]
pool_size_ = self._params["pool_size"]
pool_type_ = self._params["pool_type"]
batch_norm_ = self._params["batch_norm"]
drop_out_ = self._params["drop_out"]
num_dense_layers_ = self._params["num_dense_layers"]
final_activation_ = self._params["final_activation"]
loss_ = self._params["loss"]
optimizer_ = self._params["optimizer"]
dense_units_ = self._params["dense_units"]
metric_ = self._params["metric"]
num_gpus_ = self._params["num_gpus"]
# Input Layer
sample_shape = self._X.shape[1:]
input_layer = tfkr_layers.Input(sample_shape)
# Batch Normalization
if batch_norm_ == "before_conv3d":
norm_input_layer = tfkr_layers.BatchNormalization()(input_layer)
# First convoluton and pooling layers
conv_layer = tfkr_layers.Conv3D(
filters=kernels_per_layer_[0],
kernel_size=kernel_size_,
activation=activation_,
data_format=data_format_,
kernel_initializer="glorot_normal",
)(norm_input_layer)
else:
conv_layer = tfkr_layers.Conv3D(
filters=kernels_per_layer_[0],
kernel_size=kernel_size_,
activation=activation_,
data_format=data_format_,
kernel_initializer="glorot_normal",
)(input_layer)
if pool_type_ == "Max":
pool_layer = tfkr_layers.MaxPool3D(
pool_size=pool_size_, data_format=data_format_)(conv_layer)
elif pool_type_ == "Avg":
pool_layer = tfkr_layers.AveragePooling3D(
pool_size=pool_size_, data_format=data_format_)(conv_layer)
else:
print("Pooling layer not supported ", pool_type_)
sys.exit()
# Remaining convolution and pooling layers
for layer_idx in range(1, len(kernels_per_layer_)):
conv_layer = tfkr_layers.Conv3D(
filters=kernels_per_layer_[layer_idx],
kernel_size=kernel_size_,
activation=activation_,
data_format=data_format_,
kernel_initializer="glorot_normal",
)(pool_layer)
pool_layer = tfkr_layers.MaxPool3D(
pool_size=pool_size_, data_format=data_format_)(conv_layer)
# Batch Normalization
if batch_norm_ == "after_conv3d":
pool_layer = tfkr_layers.BatchNormalization()(pool_layer)
# Flatten
flat_layer = tfkr_layers.Flatten()(pool_layer)
# dense/dropout layers
dense_layer = tfkr_layers.Dense(
units=dense_units_,
activation=activation_,
kernel_initializer="glorot_normal",
)(flat_layer)
if drop_out_:
dense_layer = tfkr_layers.Dropout(0.4)(dense_layer)
for layer_idx in range(1, num_dense_layers_):
dense_layer = tfkr_layers.Dense(
units=dense_units_,
activation=activation_,
kernel_initializer="glorot_normal",
)(dense_layer)
if drop_out_:
dense_layer = tfkr_layers.Dropout(0.4)(dense_layer)
# output layer, sigmoid for binary classificaiton
output_layer = tfkr_layers.Dense(
units=1, activation=final_activation_)(dense_layer)
# Return model
model = tf.keras.Model(inputs=input_layer, outputs=output_layer)
if num_gpus_ > 1:
model = tf.keras.utils.multi_gpu_model(model, gpus=num_gpus_)
model.compile(loss=loss_, optimizer=optimizer_, metrics=[metric_])
return model
def to_real_keras_layer(stub_layer):
if isinstance(stub_layer, StubConv1d):
return layers.Conv1D(stub_layer.filters,
stub_layer.kernel_size,
strides=stub_layer.stride,
input_shape=stub_layer.input.shape,
padding='same') # padding
elif isinstance(stub_layer, StubConv2d):
return layers.Conv2D(stub_layer.filters,
stub_layer.kernel_size,
strides=stub_layer.stride,
input_shape=stub_layer.input.shape,
padding='same') # padding
elif isinstance(stub_layer, StubConv3d):
return layers.Conv3D(stub_layer.filters,
stub_layer.kernel_size,
strides=stub_layer.stride,
input_shape=stub_layer.input.shape,
padding='same') # padding
# TODO: Spatial Dropout
elif isinstance(stub_layer, (StubDropout1d, StubDropout2d, StubDropout3d)):
return layers.Dropout(stub_layer.rate)
# elif isinstance(stub_layer, StubDropout2d):
# return layers.SpatialDropout2D(stub_layer.rate)
# elif isinstance(stub_layer, StubDropout3d):
# return layers.SpatialDropout3D(stub_layer.rate)
elif isinstance(stub_layer, StubAvgPooling1d):
return layers.AveragePooling1D(stub_layer.kernel_size,
strides=stub_layer.stride)
elif isinstance(stub_layer, StubAvgPooling2d):
return layers.AveragePooling2D(stub_layer.kernel_size,
strides=stub_layer.stride)
elif isinstance(stub_layer, StubAvgPooling3d):
return layers.AveragePooling3D(stub_layer.kernel_size,
strides=stub_layer.stride)
elif isinstance(stub_layer, StubGlobalPooling1d):
return layers.GlobalAveragePooling1D()
elif isinstance(stub_layer, StubGlobalPooling2d):
return layers.GlobalAveragePooling2D()
elif isinstance(stub_layer, StubGlobalPooling3d):
return layers.GlobalAveragePooling3D()
elif isinstance(stub_layer, StubPooling1d):
return layers.MaxPooling1D(stub_layer.kernel_size,
strides=stub_layer.stride)
elif isinstance(stub_layer, StubPooling2d):
return layers.MaxPooling2D(stub_layer.kernel_size,
strides=stub_layer.stride)
elif isinstance(stub_layer, StubPooling3d):
return layers.MaxPooling3D(stub_layer.kernel_size,
strides=stub_layer.stride)
elif isinstance(stub_layer,
(StubBatchNormalization1d, StubBatchNormalization2d,
StubBatchNormalization3d)):
return layers.BatchNormalization(input_shape=stub_layer.input.shape)
elif isinstance(stub_layer, StubSoftmax):
return layers.Activation('softmax')
elif isinstance(stub_layer, StubReLU):
return layers.Activation('relu')
elif isinstance(stub_layer, StubFlatten):
return layers.Flatten()
elif isinstance(stub_layer, StubAdd):
return layers.Add()
elif isinstance(stub_layer, StubConcatenate):
return layers.Concatenate()
elif isinstance(stub_layer, StubDense):
return layers.Dense(stub_layer.units,
input_shape=(stub_layer.input_units, ))
def Inception_Inflated3d(include_top=True,
weights=None,
input_tensor=None,
input_shape=None,
dropout_prob=0.0,
endpoint_logit=True,
classes=400):
"""Instantiates the Inflated 3D Inception v1 architecture.
Optionally loads weights pre-trained
on Kinetics. Note that when using TensorFlow,
for best performance you should set
`image_data_format='channels_last'` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input frame(image) size for this model is 224x224.
# Arguments
include_top: whether to include the the classification
layer at the top of the network.
weights: one of `None` (random initialization)
or 'kinetics_only' (pre-training on Kinetics dataset only).
or 'imagenet_and_kinetics' (pre-training on ImageNet and Kinetics datasets).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(NUM_FRAMES, 224, 224, 3)` (with `channels_last` data format)
or `(NUM_FRAMES, 3, 224, 224)` (with `channels_first` data format).
It should have exactly 3 inputs channels.
NUM_FRAMES should be no smaller than 8. The authors used 64
frames per example for training and testing on kinetics dataset
Also, Width and height should be no smaller than 32.
E.g. `(64, 150, 150, 3)` would be one valid value.
dropout_prob: optional, dropout probability applied in dropout layer
after global average pooling layer.
0.0 means no dropout is applied, 1.0 means dropout is applied to all features.
Note: Since Dropout is applied just before the classification
layer, it is only useful when `include_top` is set to True.
endpoint_logit: (boolean) optional. If True, the model's forward pass
will end at producing logits. Otherwise, softmax is applied after producing
the logits to produce the class probabilities prediction. Setting this parameter
to True is particularly useful when you want to combine results of rgb model
and optical flow model.
- `True` end model forward pass at logit output
- `False` go further after logit to produce softmax predictions
Note: This parameter is only useful when `include_top` is set to True.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in WEIGHTS_NAME or weights is None
or os.path.exists(weights)):
raise ValueError(
'The `weights` argument should be either '
'`None` (random initialization) or %s' % str(WEIGHTS_NAME) + ' '
'or a valid path to a file containing `weights` values')
if weights in WEIGHTS_NAME and include_top and classes != 400:
raise ValueError(
'If using `weights` as one of these %s, with `include_top`'
' as true, `classes` should be 400' % str(WEIGHTS_NAME))
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_frame_size=224,
min_frame_size=32,
default_num_frames=64,
min_num_frames=8,
data_format=B.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = K.Input(shape=input_shape)
else:
if not B.is_keras_tensor(input_tensor):
img_input = K.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if B.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 4
# Downsampling via convolution (spatial and temporal)
x = conv3d_bn(img_input,
64,
7,
7,
7,
strides=(2, 2, 2),
padding='same',
name='Conv3d_1a_7x7')
# Downsampling (spatial only)
x = L.MaxPooling3D((1, 3, 3),
strides=(1, 2, 2),
padding='same',
name='MaxPool2d_2a_3x3')(x)
x = conv3d_bn(x,
64,
1,
1,
1,
strides=(1, 1, 1),
padding='same',
name='Conv3d_2b_1x1')
x = conv3d_bn(x,
192,
3,
3,
3,
strides=(1, 1, 1),
padding='same',
name='Conv3d_2c_3x3')
# Downsampling (spatial only)
x = L.MaxPooling3D((1, 3, 3),
strides=(1, 2, 2),
padding='same',
name='MaxPool2d_3a_3x3')(x)
# Mixed 3b
branch_0 = conv3d_bn(x,
64,
1,
1,
1,
padding='same',
name='Conv3d_3b_0a_1x1')
branch_1 = conv3d_bn(x,
96,
1,
1,
1,
padding='same',
name='Conv3d_3b_1a_1x1')
branch_1 = conv3d_bn(branch_1,
128,
3,
3,
3,
padding='same',
name='Conv3d_3b_1b_3x3')
branch_2 = conv3d_bn(x,
16,
1,
1,
1,
padding='same',
name='Conv3d_3b_2a_1x1')
branch_2 = conv3d_bn(branch_2,
32,
3,
3,
3,
padding='same',
name='Conv3d_3b_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_3b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
32,
1,
1,
1,
padding='same',
name='Conv3d_3b_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_3b')
# Mixed 3c
branch_0 = conv3d_bn(x,
128,
1,
1,
1,
padding='same',
name='Conv3d_3c_0a_1x1')
branch_1 = conv3d_bn(x,
128,
1,
1,
1,
padding='same',
name='Conv3d_3c_1a_1x1')
branch_1 = conv3d_bn(branch_1,
192,
3,
3,
3,
padding='same',
name='Conv3d_3c_1b_3x3')
branch_2 = conv3d_bn(x,
32,
1,
1,
1,
padding='same',
name='Conv3d_3c_2a_1x1')
branch_2 = conv3d_bn(branch_2,
96,
3,
3,
3,
padding='same',
name='Conv3d_3c_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_3c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_3c_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_3c')
# Downsampling (spatial and temporal)
x = L.MaxPooling3D((3, 3, 3),
strides=(2, 2, 2),
padding='same',
name='MaxPool2d_4a_3x3')(x)
# Mixed 4b
branch_0 = conv3d_bn(x,
192,
1,
1,
1,
padding='same',
name='Conv3d_4b_0a_1x1')
branch_1 = conv3d_bn(x,
96,
1,
1,
1,
padding='same',
name='Conv3d_4b_1a_1x1')
branch_1 = conv3d_bn(branch_1,
208,
3,
3,
3,
padding='same',
name='Conv3d_4b_1b_3x3')
branch_2 = conv3d_bn(x,
16,
1,
1,
1,
padding='same',
name='Conv3d_4b_2a_1x1')
branch_2 = conv3d_bn(branch_2,
48,
3,
3,
3,
padding='same',
name='Conv3d_4b_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_4b_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4b')
# Mixed 4c
branch_0 = conv3d_bn(x,
160,
1,
1,
1,
padding='same',
name='Conv3d_4c_0a_1x1')
branch_1 = conv3d_bn(x,
112,
1,
1,
1,
padding='same',
name='Conv3d_4c_1a_1x1')
branch_1 = conv3d_bn(branch_1,
224,
3,
3,
3,
padding='same',
name='Conv3d_4c_1b_3x3')
branch_2 = conv3d_bn(x,
24,
1,
1,
1,
padding='same',
name='Conv3d_4c_2a_1x1')
branch_2 = conv3d_bn(branch_2,
64,
3,
3,
3,
padding='same',
name='Conv3d_4c_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_4c_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4c')
# Mixed 4d
branch_0 = conv3d_bn(x,
128,
1,
1,
1,
padding='same',
name='Conv3d_4d_0a_1x1')
branch_1 = conv3d_bn(x,
128,
1,
1,
1,
padding='same',
name='Conv3d_4d_1a_1x1')
branch_1 = conv3d_bn(branch_1,
256,
3,
3,
3,
padding='same',
name='Conv3d_4d_1b_3x3')
branch_2 = conv3d_bn(x,
24,
1,
1,
1,
padding='same',
name='Conv3d_4d_2a_1x1')
branch_2 = conv3d_bn(branch_2,
64,
3,
3,
3,
padding='same',
name='Conv3d_4d_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4d_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_4d_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4d')
# Mixed 4e
branch_0 = conv3d_bn(x,
112,
1,
1,
1,
padding='same',
name='Conv3d_4e_0a_1x1')
branch_1 = conv3d_bn(x,
144,
1,
1,
1,
padding='same',
name='Conv3d_4e_1a_1x1')
branch_1 = conv3d_bn(branch_1,
288,
3,
3,
3,
padding='same',
name='Conv3d_4e_1b_3x3')
branch_2 = conv3d_bn(x,
32,
1,
1,
1,
padding='same',
name='Conv3d_4e_2a_1x1')
branch_2 = conv3d_bn(branch_2,
64,
3,
3,
3,
padding='same',
name='Conv3d_4e_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4e_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_4e_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4e')
# Mixed 4f
branch_0 = conv3d_bn(x,
256,
1,
1,
1,
padding='same',
name='Conv3d_4f_0a_1x1')
branch_1 = conv3d_bn(x,
160,
1,
1,
1,
padding='same',
name='Conv3d_4f_1a_1x1')
branch_1 = conv3d_bn(branch_1,
320,
3,
3,
3,
padding='same',
name='Conv3d_4f_1b_3x3')
branch_2 = conv3d_bn(x,
32,
1,
1,
1,
padding='same',
name='Conv3d_4f_2a_1x1')
branch_2 = conv3d_bn(branch_2,
128,
3,
3,
3,
padding='same',
name='Conv3d_4f_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4f_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
128,
1,
1,
1,
padding='same',
name='Conv3d_4f_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4f')
# Downsampling (spatial and temporal)
x = L.MaxPooling3D((2, 2, 2),
strides=(2, 2, 2),
padding='same',
name='MaxPool2d_5a_2x2')(x)
# Mixed 5b
branch_0 = conv3d_bn(x,
256,
1,
1,
1,
padding='same',
name='Conv3d_5b_0a_1x1')
branch_1 = conv3d_bn(x,
160,
1,
1,
1,
padding='same',
name='Conv3d_5b_1a_1x1')
branch_1 = conv3d_bn(branch_1,
320,
3,
3,
3,
padding='same',
name='Conv3d_5b_1b_3x3')
branch_2 = conv3d_bn(x,
32,
1,
1,
1,
padding='same',
name='Conv3d_5b_2a_1x1')
branch_2 = conv3d_bn(branch_2,
128,
3,
3,
3,
padding='same',
name='Conv3d_5b_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_5b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
128,
1,
1,
1,
padding='same',
name='Conv3d_5b_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_5b')
# Mixed 5c
branch_0 = conv3d_bn(x,
384,
1,
1,
1,
padding='same',
name='Conv3d_5c_0a_1x1')
branch_1 = conv3d_bn(x,
192,
1,
1,
1,
padding='same',
name='Conv3d_5c_1a_1x1')
branch_1 = conv3d_bn(branch_1,
384,
3,
3,
3,
padding='same',
name='Conv3d_5c_1b_3x3')
branch_2 = conv3d_bn(x,
48,
1,
1,
1,
padding='same',
name='Conv3d_5c_2a_1x1')
branch_2 = conv3d_bn(branch_2,
128,
3,
3,
3,
padding='same',
name='Conv3d_5c_2b_3x3')
branch_3 = L.MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_5c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
128,
1,
1,
1,
padding='same',
name='Conv3d_5c_3b_1x1')
x = L.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_5c')
if include_top:
# Classification block
x = L.AveragePooling3D((2, 7, 7),
strides=(1, 1, 1),
padding='valid',
name='global_avg_pool')(x)
x = L.Dropout(dropout_prob)(x)
x = conv3d_bn(x,
classes,
1,
1,
1,
padding='same',
use_bias=True,
use_activation_fn=False,
use_bn=False,
name='Conv3d_6a_1x1')
num_frames_remaining = int(x.shape[1])
x = L.Reshape((num_frames_remaining, classes))(x)
# logits (raw scores for each class)
x = L.Lambda(lambda x: B.mean(x, axis=1, keepdims=False),
output_shape=lambda s: (s[0], s[2]))(x)
if not endpoint_logit:
x = L.Activation('softmax', name='prediction')(x)
else:
h = int(x.shape[2])
w = int(x.shape[3])
x = L.AveragePooling3D((2, h, w),
strides=(1, 1, 1),
padding='valid',
name='global_avg_pool')(x)
inputs = img_input
# create model
model = K.Model(inputs, x, name='i3d_inception')
# load weights
if weights in WEIGHTS_NAME:
if weights == WEIGHTS_NAME[0]: # rgb_kinetics_only
if include_top:
weights_url = WEIGHTS_PATH['rgb_kinetics_only']
model_name = 'i3d_inception_rgb_kinetics_only.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['rgb_kinetics_only']
model_name = 'i3d_inception_rgb_kinetics_only_no_top.h5'
elif weights == WEIGHTS_NAME[1]: # flow_kinetics_only
if include_top:
weights_url = WEIGHTS_PATH['flow_kinetics_only']
model_name = 'i3d_inception_flow_kinetics_only.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['flow_kinetics_only']
model_name = 'i3d_inception_flow_kinetics_only_no_top.h5'
elif weights == WEIGHTS_NAME[2]: # rgb_imagenet_and_kinetics
if include_top:
weights_url = WEIGHTS_PATH['rgb_imagenet_and_kinetics']
model_name = 'i3d_inception_rgb_imagenet_and_kinetics.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['rgb_imagenet_and_kinetics']
model_name = 'i3d_inception_rgb_imagenet_and_kinetics_no_top.h5'
elif weights == WEIGHTS_NAME[3]: # flow_imagenet_and_kinetics
if include_top:
weights_url = WEIGHTS_PATH['flow_imagenet_and_kinetics']
model_name = 'i3d_inception_flow_imagenet_and_kinetics.h5'
else:
weights_url = WEIGHTS_PATH_NO_TOP['flow_imagenet_and_kinetics']
model_name = 'i3d_inception_flow_imagenet_and_kinetics_no_top.h5'
downloaded_weights_path = get_file(model_name,
weights_url,
cache_subdir='models')
model.load_weights(downloaded_weights_path)
if B.image_data_format() == 'channels_first' and B.backend(
) == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your keras config '
'at ~/.keras/keras.json.')
elif weights is not None:
model.load_weights(weights)
return model
def resnet(input_shape, depth, num_classes, use3D=False, useBatchNorm=True):
"""ResNet Version 1 Model builder [a]
Stacks of 2 x (3 x 3) Conv2D-BN-ReLU
Last ReLU is after the shortcut connection.
At the beginning of each stage, the feature map size is downsampled
by a convolutional layer with strides=2, while the number of filters is
doubled. Within each stage, the layers have the same number filters and the
same number of filters.
Features maps sizes:
stage 0: 32x32, 16
stage 1: 16x16, 32
stage 2: 8x8, 64
The Number of parameters is approx the same as Table 6 of [a]:
ResNet20 0.27M
ResNet32 0.46M
ResNet44 0.66M
ResNet56 0.85M
ResNet110 1.7M
# Arguments
input_shape (tensor): shape of input image tensor
depth (int): number of core convolutional layers
num_classes (int): number of classes (CIFAR10 has 10)
# Returns
model (Model): Keras model instance
"""
if (depth - 2) % 6 != 0:
raise ValueError('depth should be 6n+2 (eg 20, 32, 44 in [a])')
# Start model definition.
num_filters = 16
num_res_blocks = int((depth - 2) / 6)
inputs = layers.Input(shape=input_shape)
x = resnet_layer(inputs=inputs, use3D=use3D)
# Instantiate the stack of residual units
for stack in range(3):
for res_block in range(num_res_blocks):
strides = 1
if stack > 0 and res_block == 0: # first layer but not first stack
strides = 2 # downsample
y = resnet_layer(inputs=x,
num_filters=num_filters,
strides=strides,
batch_normalization=useBatchNorm,
use3D=use3D)
y = resnet_layer(inputs=y,
num_filters=num_filters,
activation=None,
batch_normalization=useBatchNorm,
use3D=use3D)
if stack > 0 and res_block == 0: # first layer but not first stack
# linear projection residual shortcut connection to match
# changed dims
x = resnet_layer(inputs=x,
num_filters=num_filters,
kernel_size=1,
strides=strides,
activation=None,
batch_normalization=False,
use3D=use3D)
x = layers.add([x, y])
x = layers.Activation('relu')(x)
num_filters *= 2
# Add classifier on top.
# v1 does not use BN after last shortcut connection-ReLU
if use3D:
x = layers.AveragePooling3D(pool_size=8)(x)
else:
x = layers.AveragePooling2D(pool_size=8)(x)
y = layers.Flatten()(x)
outputs = layers.Dense(num_classes,
activation='softmax',
kernel_initializer='he_normal')(y)
# Instantiate model.
model = keras.Model(inputs=inputs, outputs=outputs)
return model
def ClassificationLayer(inputs, num_classes):
x = layers.AveragePooling3D(pool_size=(2,7,7),
strides=(2,1,1))(inputs)
x = layers.Dense(num_classes, activation='softmax')(x)
return x
# --- Create model inputs
inputs = client.get_inputs(Input)
# --- Define lambda functions
conv3 = lambda x, filters: layers.Conv3D(kernel_size=(1, 3, 3),
filters=filters,
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(
0.01))(x)
conv1 = lambda x, filters: layers.Conv3D(
kernel_size=(1, 1, 1), filters=filters, strides=1, padding='same')(x)
pool = lambda x: layers.AveragePooling3D(
pool_size=(1, 2, 2), strides=(1, 2, 2), padding='same')(x)
norm = lambda x: layers.BatchNormalization()(x)
relu = lambda x: layers.LeakyReLU()(x)
concat = lambda a, b: layers.Concatenate()([a, b])
#drop = lambda x : layers.Dropout(0.2)(x)
dense = lambda x, k: conv3(relu(norm(x)), filters=k)
bneck = lambda x, b: conv1(relu(norm(x)), filters=b)
def dense_block(x, k=8, n=3, b=1):
ds_layer = None
for i in range(n):
cc_layer = concat(cc_layer, ds_layer) if ds_layer is not None else x
