def create_pyramid_level(backbone_input,
upsamplelike_input=None,
addition_input=None,
level=5,
ndim=2,
feature_size=256):
"""Create a pyramid layer from a particular backbone input layer.
Args:
backbone_input (layer): Backbone layer to use to create they pyramid layer
upsamplelike_input ([type], optional): Defaults to None. Input to use
as a template for shape to upsample to
addition_input (layer, optional): Defaults to None. Layer to add to
pyramid layer after conv and upsample
level (int, optional): Defaults to 5. Level to use in layer names
feature_size (int, optional): Defaults to 256. Number of filters for
convolutional layer
ndim: The spatial dimensions of the input data. Default is 2,
but it also works with 3
Returns:
(pyramid final, pyramid upsample): Pyramid layer after processing,
upsampled pyramid layer
"""
acceptable_ndims = {2, 3}
if ndim not in acceptable_ndims:
raise ValueError('Only 2 and 3 dimensional networks are supported')
reduced_name = 'C%s_reduced' % level
upsample_name = 'P%s_upsampled' % level
addition_name = 'P%s_merged' % level
final_name = 'P%s' % level
# Apply 1x1 conv to backbone layer
if ndim == 2:
pyramid = Conv2D(feature_size, (1, 1), strides=(1, 1),
padding='same', name=reduced_name)(backbone_input)
else:
pyramid = Conv3D(feature_size, (1, 1, 1), strides=(1, 1, 1),
padding='same', name=reduced_name)(backbone_input)
# Upsample pyramid input
if upsamplelike_input is not None:
pyramid_upsample = UpsampleLike(name=upsample_name)(
[pyramid, upsamplelike_input])
else:
pyramid_upsample = None
# Add and then 3x3 conv
if addition_input is not None:
pyramid = Add(name=addition_name)([pyramid, addition_input])
if ndim == 2:
pyramid_final = Conv2D(feature_size, (3, 3), strides=(1, 1),
padding='same', name=final_name)(pyramid)
else:
pyramid_final = Conv3D(feature_size, (3, 3, 3), strides=(1, 1, 1),
padding='same', name=final_name)(pyramid)
return pyramid_final, pyramid_upsample
def box_convolution(input, box_filters, l2):
reduce = Conv3D(box_filters, 1, kernel_regularizer=l2)(input)
squash = Conv3D(box_filters, (FOUR, FOUR, FOUR), kernel_regularizer=l2)(reduce)
gather = Reshape((box_filters,))(squash)
repeat = RepeatVector(FOUR * FOUR * FOUR)(gather)
spread = Reshape((FOUR, FOUR, FOUR, box_filters))(repeat)
return spread
def _conv_block(x, filters, bottleneck):
bn_scale = PARAMS['bn_scale']
activation = PARAMS['activation']
kernel_initializer = PARAMS['kernel_initializer']
weight_decay = PARAMS['weight_decay']
dropout_rate = PARAMS['dropout_rate']
x = BatchNormalization(scale=bn_scale, axis=-1)(x)
x = activation()(x)
x = Conv3D(filters * bottleneck,
kernel_size=(1, 1, 1),
padding='same',
use_bias=False,
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_penalty(weight_decay))(x)
if dropout_rate is not None:
x = SpatialDropout3D(dropout_rate)(x)
x = BatchNormalization(scale=bn_scale, axis=-1)(x)
x = activation()(x)
x = Conv3D(filters,
kernel_size=(3, 3, 3),
padding='same',
use_bias=True,
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_penalty(weight_decay))(x)
return x
def build_rim(params):
nc = params['nc']
input_layer = Conv3D(params['input_size'],
kernel_size=params['input_kernel_size'],
trainable=True,
padding='SAME',
input_shape=(None, nc, nc, nc, 2),
activation=params['input_activation'])
cell = ConvLSTM3DCell(params['cell_size'],
kernel_size=params['cell_kernel_size'],
padding='SAME')
cell.build(input_shape=[None, nc, nc, nc, params['input_size']])
output_layer = Conv3D(1,
kernel_size=params['output_kernel_size'],
trainable=True,
padding='SAME',
input_shape=(None, nc, nc, nc, params['cell_size']),
activation=params['output_activation'])
rim = RIM3D(cell, input_layer, output_layer, niter=params['rim_iter'])
return rim
def create_model(self):
self.model = Sequential()
self.model.add(Conv3D(filters= 10, kernel_size=(4,10,10), input_shape =(4,100,100,1), data_format="channels_last", strides=(1,5,5), activation="relu"))
self.model.add(Conv3D(filters= 32, kernel_size=(1,5,5), strides=(1,5,5),data_format="channels_last", activation="relu"))
self.model.add(Flatten())
self.model.add(Dense(256, activation="relu"))
self.model.add(Dense(3, activation='softmax'))
def build_rim_parallel(params):
nc = params['nc']
input_layer = Conv3D(params['input_size'], kernel_size=params['input_kernel_size'],
trainable=True, padding='SAME',
input_shape=(None, nc, nc, nc, 2), activation=params['input_activation'])
input_layer_sub = Conv3D(params['input_size'], kernel_size=params['input_kernel_size'],
trainable=True, padding='SAME', strides= [params['strides']]*3,
input_shape=(None, nc, nc, nc, 2), activation=params['input_activation'])
cell1 = ConvLSTM3DCell(params['cell_size'], kernel_size=params['cell_kernel_size'], padding='SAME')
output_layer_up = Conv3DTranspose(params['cell_size'], kernel_size=params['middle_kernel_size'],
trainable=True, padding='SAME', strides=[params['strides']]*3,
activation=params['output_activation'])
cell2 = ConvLSTM3DCell(params['cell_size'], kernel_size=params['cell_kernel_size'], padding='SAME')
output_layer = Conv3D(1, kernel_size=params['output_kernel_size'], trainable=True, padding='SAME',
input_shape=(None, nc, nc, nc, params['cell_size']*2), activation=params['output_activation'])
rim = RIM3D_parallel(cell1, cell2, input_layer, input_layer_sub, output_layer_up, output_layer, strides=params['strides'],
niter=params['rim_iter'])
return rim
def create_model(input_size, filters, c=10 ** -4):
l2 = regularizers.l2(c)
input = Input(shape=(FOUR, FOUR, FOUR, input_size))
conv_1 = normalized_relu(line_convolution(input, filters, l2))
conv_2 = normalized_relu(line_convolution(conv_1, filters, l2))
conv_3 = normalized_relu(line_convolution(conv_2, filters, l2))
last_conv = Conv3D(filters, 1, kernel_regularizer=l2)(conv_3)
collapse_play = normalized_relu(Conv3D(filters // 2, (1, 1, 4), kernel_regularizer=l2)(last_conv))
squash_play = normalized_relu(Conv3D(3, 1, kernel_regularizer=l2)(collapse_play))
flatten_play = Flatten()(squash_play)
dense_play = Dense(16, activation='relu', kernel_regularizer=l2)(flatten_play)
sigmoid_play = Dense(16, activation='sigmoid', kernel_regularizer=l2)(dense_play)
output_play = Softmax()(sigmoid_play)
collapse_win = normalized_relu(Conv3D(3, (1, 1, 4), kernel_regularizer=l2)(last_conv))
flatten_win = Flatten()(collapse_win)
dense_win = Dense(16, activation='relu', kernel_regularizer=l2)(flatten_win)
output_win = Dense(1, activation='tanh', kernel_regularizer=l2)(dense_win)
model = Model(inputs=input, outputs=[output_play, output_win])
optimizer = Adam()
metics = {'softmax': 'categorical_accuracy', 'dense_3': 'mae'}
model.compile(optimizer, ['categorical_crossentropy', 'mse'], metrics=metics)
return model
def build_rim_parallel3_single(params):
nc = params['nc']
cell1 = ConvLSTM3DCell(params['cell_size1'], kernel_size=params['cell_kernel_size1'], padding='SAME')
cell2 = ConvLSTM3DCell(params['cell_size2'], kernel_size=params['cell_kernel_size2'], padding='SAME')
cell3 = ConvLSTM3DCell(params['cell_size3'], kernel_size=params['cell_kernel_size3'], padding='SAME')
cells = [cell1, cell2, cell3]
input_layer1 = Conv3D(params['input_size1'], kernel_size=params['input_kernel_size1'],
trainable=True, padding='SAME', activation=params['input_activation'])
input_layer2 = Conv3D(params['input_size2'], kernel_size=params['input_kernel_size2'], strides= [params['strides']]*3,
trainable=True, padding='SAME', activation=params['input_activation'])
input_layer3 = Conv3D(params['input_size3'], kernel_size=params['input_kernel_size3'], strides= [2*params['strides']]*3,
trainable=True, padding='SAME', activation=params['input_activation'])
input_layers = [input_layer1, input_layer2, input_layer3]
output_layer1 = Conv3D(params['output_size1'], kernel_size=params['output_kernel_size1'], trainable=True, padding='SAME',
activation=params['output_activation'])
output_layer2 = Conv3DTranspose(params['output_size2'], kernel_size=params['output_kernel_size2'], trainable=True, padding='SAME',
strides= [params['strides']]*3, activation=params['output_activation'])
output_layer3 = Conv3DTranspose(params['output_size3'], kernel_size=params['output_kernel_size3'], trainable=True, padding='SAME',
strides= [params['strides']]*3, activation=params['output_activation'])
output_layers = [output_layer1, output_layer2, output_layer3]
rim = RIM3D_parallel3_single(cells, input_layers, output_layers, strides=params['strides'], niter=params['rim_iter'])
return rim
def build_model(self):
input_net = Input(shape=(self.input_trace, util.IMG_HEIGHT,
util.IMG_WIDTH, util.INPUT_CHANNELS))
x = Conv3D(16, (7, 3, 3), padding="valid",
strides=(1, 2, 2))(input_net)
x = Activation('relu')(x)
x = Conv3D(16, (7, 3, 3), padding="valid", strides=(1, 2, 2))(x)
x = Activation('relu')(x)
x = Conv3D(32, (7, 3, 3), padding="valid", strides=(1, 2, 2))(x)
x = Activation('relu')(x)
x = Conv3D(32, (7, 3, 3), padding="valid", strides=(1, 2, 2))(x)
x = Activation('relu')(x)
# x = Conv3D(64, (25, 1, 1), padding="valid", strides=(1,1,1))(x)
# x = Activation('relu')(x)
x = Flatten()(x)
#x = Dense(1024, activation='relu')(x)
x = Dense(512, activation='relu')(x)
x = Dense(3)(x)
model = Model(inputs=input_net, outputs=x)
model.summary()
model.compile(optimizer=Adam(lr=0.001),
loss=ConvNet.root_mean_squared_error,
metrics=['mae'])
return model
def featurenet_3D_block(x, n_filters):
"""Add a set of layers that make up one unit of the featurenet 3D backbone
Args:
x (layer): Keras layer object to pass to backbone unit
n_filters (int): Number of filters to use for convolutional layers
Returns:
layer: Keras layer object
"""
df = K.image_data_format()
# conv set 1
x = Conv3D(n_filters, (3, 3, 3),
strides=(1, 1, 1),
padding='same',
data_format=df)(x)
x = BatchNormalization(axis=-1)(x)
x = Activation('relu')(x)
# conv set 2
x = Conv3D(n_filters, (3, 3, 3),
strides=(1, 1, 1),
padding='same',
data_format=df)(x)
x = BatchNormalization(axis=-1)(x)
x = Activation('relu')(x)
# Final max pooling stage
x = MaxPool3D(pool_size=(2, 2, 2), data_format=df)(x)
return x
def space_attention_block(input, filters, kernel_size):
output_trunk = input
x = Conv3D(filters=filters, kernel_size=kernel_size, padding='same', use_bias=False,
kernel_initializer='he_normal', kernel_regularizer=l2(5e-4))(input)
x = BatchNormalization(axis=-1)(x)
x = Activation('relu')(x)
x_1 = Conv3D(filters, kernel_size=kernel_size, strides=(2, 2, 1), padding='same')(x)
x_1 = Activation('relu')(x_1)
x_2 = Conv3D(filters * 2, kernel_size=kernel_size, strides=(2, 2, 1), padding='same')(x_1)
x_2 = Activation('relu')(x_2)
x_3 = Conv3DTranspose(filters=filters, kernel_size=kernel_size, strides=(2, 2, 1), padding='same')(x_2)
x_3 = Activation('relu')(x_3)
x_4 = Conv3DTranspose(filters=filters, kernel_size=kernel_size, strides=(2, 2, 1), padding='same')(x_3)
x_4 = Activation('sigmoid')(x_4)
# x_4 = Activation('relu')(x_4)
output = Multiply()([x_4, x])
# output = add([output_trunk, x_4])
# output = Lambda(lambda x: x + 1)(x_4)
# output = Multiply()([output, output_trunk])
x_add = add([output, output_trunk])
return x_add
def main(
batch_size=16,
episode_length=16,
filters=16,
width=64,
height=64,
memory_size=32,
):
# Prevent TensorFlow from allocating all available GPU memory
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
tf.keras.backend.set_session(tf.Session(config=config))
input_layer = Input([episode_length, width, height, 1])
layer = input_layer
layer = Conv3D(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3D(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3D(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3D(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3D(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
tmp_shape = layer.shape.as_list()[1:]
code_size = tmp_shape[1] * tmp_shape[2] * tmp_shape[3]
layer = Reshape([episode_length, code_size])(layer)
layer = Memory(code_size=code_size, memory_size=memory_size)(layer)
layer = Reshape(tmp_shape)(layer)
layer = Conv3DTranspose(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3DTranspose(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3DTranspose(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3DTranspose(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3DTranspose(filters=filters, kernel_size=3, strides=(1, 2, 2), padding="same")(layer)
layer = Conv3DTranspose(filters=1, kernel_size=1, strides=1, padding="same", activation="sigmoid")(layer)
output_layer = layer
model = Model(inputs=input_layer, outputs=output_layer)
model.compile("adam", loss="mse", metrics=["mse"])
model.summary()
for var in model.variables:
print(var, end=' ')
print(var.trainable)
dataset_input_tensor = tf.random.normal(shape=[episode_length, width, height, 1])
dataset_input_tensor = tf.clip_by_value(dataset_input_tensor, 0.0, 1.0)
dataset = tf.data.Dataset.from_tensors(dataset_input_tensor)
dataset = dataset.repeat(-1)
dataset = dataset.map(lambda x: (x, x))
dataset = dataset.batch(batch_size)
log_dir = "../logs/KanervaMachine/log_{}".format(int(time()))
os.makedirs(log_dir)
tensorboard = TensorBoard(log_dir=log_dir, update_freq="batch")
model.fit(dataset, callbacks=[tensorboard], steps_per_epoch=500, epochs=100)
def default_classification_model(num_classes,
num_anchors,
pyramid_feature_size=256,
prior_probability=0.01,
classification_feature_size=256,
name='classification_submodel'):
"""Creates the default regression submodel.
Args:
num_classes: Number of classes to predict a score for at each feature level.
num_anchors: Number of anchors to predict classification
scores for at each feature level.
pyramid_feature_size: The number of filters to expect from the
feature pyramid levels.
classification_feature_size: The number of filters to use in the layers
in the classification submodel.
name: The name of the submodel.
Returns:
A keras.models.Model that predicts classes for each anchor.
"""
options = {
'kernel_size': 3,
'strides': 1,
'padding': 'same',
}
if K.image_data_format() == 'channels_first':
inputs = Input(shape=(pyramid_feature_size, None, None, None))
else:
inputs = Input(shape=(None, None, None, pyramid_feature_size))
outputs = inputs
for i in range(4):
outputs = Conv3D(filters=classification_feature_size,
activation='relu',
name='pyramid_classification_{}'.format(i),
kernel_initializer=RandomNormal(mean=0.0,
stddev=0.01,
seed=None),
bias_initializer='zeros',
**options)(outputs)
outputs = Conv3D(
filters=num_classes * num_anchors,
kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None),
bias_initializer=PriorProbability(probability=prior_probability),
name='pyramid_classification',
**options)(outputs)
# reshape output and apply sigmoid
if K.image_data_format() == 'channels_first':
outputs = Permute((2, 3, 1),
name='pyramid_classification_permute')(outputs)
outputs = Reshape((-1, num_classes),
name='pyramid_classification_reshape')(outputs)
outputs = Activation('sigmoid',
name='pyramid_classification_sigmoid')(outputs)
return Model(inputs=inputs, outputs=outputs, name=name)
def __init__(self,
units,
input_size,
niter,
kbinmap,
args,
droprate=0.05,
regrate=1e-5):
super(RIMHybrid, self).__init__()
self.nc, self.bs = args.nc, args.bs
nc, bs = args.nc, args.bs
self.niter = niter
print("number of iterations : ", niter)
self.beta_1, self.beta_2 = 0.9, 0.999
self.lr = 0.1
self.eps = 1e-7
self.kbinmap = tf.constant(kbinmap)
self.kbinmapflat = tf.constant(kbinmap.flatten())
self.layers_in = [
Dense(units, activation='tanh', kernel_regularizer=l2reg(regrate))
for i in range(niter)
]
self.drop_in = [Dropout(rate=droprate) for i in range(niter)]
self.layers_out = [
Dense(2 * nc,
activation='linear',
kernel_regularizer=l2reg(regrate)) for i in range(niter)
]
self.drop_out = [Dropout(rate=droprate) for i in range(niter)]
self.conv_in = [
Conv1D(1, 1, activation='linear') for i in range(niter)
]
self.conv_out = [Conv1D(1, 1) for i in range(niter)]
self.conv_relu = [
Conv1D(1, 1, activation='relu') for i in range(niter)
]
self.layers1d = [[self.layers_in[i], self.layers_out[i], \
self.drop_in[i], self.drop_out[i],
self.conv_in[i], self.conv_out[i],
self.conv_relu[i]] for i in range(niter)]
cell_size = input_size
self.input_layer = Conv3D(input_size,
kernel_size=5,
trainable=True,
padding='SAME',
input_shape=(None, nc, nc, nc, 2),
activation='tanh')
self.cell = ConvLSTM3DCell(cell_size, kernel_size=5, padding='SAME')
self.output_layer = Conv3D(1,
kernel_size=5,
trainable=True,
padding='SAME',
input_shape=(None, nc, nc, nc, cell_size))
def transform_layer(self, x, strides, scope):
with tf.name_scope(scope):
x = Conv3D(filters=self.depth, kernel_size=(1, 1, 1), strides=(1, 1, 1), padding='same')(x)
x = BatchNormalization(axis=-1)(x, training=self.training)
x = Activation("relu")(x)
x = Conv3D(filters=self.depth, kernel_size=(3, 3, 3), strides=strides, padding='same')(x)
x = BatchNormalization(axis=-1)(x, training=self.training)
x = Activation("relu")(x)
return x
def conv_block(input_tensor, kernel_size, filters, stage, block, path,strides=(1, 2 ,2),non_degenerate_temporal_conv=False):
"""A block that has a conv layer at shortcut.
Arguments:
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
strides: Strides for the first conv layer in the block.
Returns:
Output tensor for the block.
Note that from stage 3,
the first conv layer at main path is with strides=(2, 2)
And the shortcut should have strides=(2, 2) as well
"""
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 4
else:
bn_axis = 1
conv_name_base = str(path) + 'res' + str(stage) + block + '_branch'
bn_name_base = str(path) + 'bn' + str(stage) + block + '_branch'
if non_degenerate_temporal_conv == True:
x = Conv3D(
filters1, (3, 1, 1), strides=strides, padding='same', kernel_regularizer=l2(1e-4), name=conv_name_base + '2a')(
input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
else:
x = Conv3D(
filters1, (1, 1, 1), strides=strides, kernel_regularizer=l2(1e-4), name=conv_name_base + '2a')(
input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv3D(
filters2, kernel_size, padding='same', kernel_regularizer=l2(1e-4), name=conv_name_base + '2b')(
x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv3D(filters3, (1, 1 ,1), kernel_regularizer=l2(1e-4), name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
shortcut = Conv3D(
filters3, (1, 1, 1), strides=strides, kernel_regularizer=l2(1e-4), name=conv_name_base + '1')(
input_tensor)
shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)
x = layers.add([x, shortcut])
x = Activation('relu')(x)
return x
def modelForward(self, in_cen, in_per, is_train=True):
#def probTrain():
# return self.__do_prob
#def probValid():
# return 1.0
#do_prob = tf.cond(tf.equal(is_train, tf.constant(True, dtype=tf.bool)), probTrain, probValid)
with tf.variable_scope('model_scope', reuse=tf.AUTO_REUSE) as scope:
## 1 conv -> batch norm -> activate
conv1 = Conv3D(self.__filters[0],
kernel_size=self.__kernels[0],
strides=self.__strides[0],
padding=self.__paddings[0])(in_cen)
conv1 = tf.contrib.layers.batch_norm(conv1,
is_training=is_train,
epsilon=1e-5,
decay=0.9,
updates_collections=None)
#conv1 = BatchNormalization(momentum=0.9, epsilon=1e-5)(conv1, training=is_train)
conv1 = LeakyReLU(alpha=0.2)(conv1)
## pad act1 and then add it to in_per (to the peripheral modules)
cen_paddings = tf.constant([[0, 0], [1, 1], [1, 1], [0, 0],
[0, 0]]) # pad only the cross-section
conv1_padded = tf.pad(conv1, cen_paddings)
conv1_combined = tf.add(conv1_padded, in_per)
## 2 conv -> batch norm -> activate
conv2 = Conv3D(self.__filters[1],
kernel_size=self.__kernels[1],
strides=self.__strides[1],
padding=self.__paddings[1])(conv1_combined)
conv2 = tf.contrib.layers.batch_norm(conv2,
is_training=is_train,
epsilon=1e-5,
decay=0.9,
updates_collections=None)
#conv2 = BatchNormalization(momentum=0.9, epsilon=1e-5)(conv2, training=is_train)
conv2 = LeakyReLU(alpha=0.2)(conv2)
## 1 FC
fc1 = Flatten()(conv2)
fc1 = Dense(self.__fc_dim[0], kernel_regularizer=l2())(fc1)
fc1 = LeakyReLU(alpha=0.3)(fc1)
## Dropout
fc1_drop = Dropout(self.__do_prob)(fc1, training=is_train)
## 2 FC
logits = Dense(self._n_classes, kernel_regularizer=l2())(fc1_drop)
return logits
def semantic_upsample(x, n_upsample, n_filters=64, ndim=2, target=None):
"""
Performs iterative rounds of 2x upsampling and
convolutions with a 3x3 filter to remove aliasing effects
Args:
x (tensor): The input tensor to be upsampled
n_upsample (int): The number of 2x upsamplings
n_filters (int, optional): Defaults to 256. The number of filters for
the 3x3 convolution
target (tensor, optional): Defaults to None. A tensor with the target
shape. If included, then the final upsampling layer will reshape
to the target tensor's size
ndim: The spatial dimensions of the input data.
Default is 2, but it also works with 3
Returns:
The upsampled tensor
"""
acceptable_ndims = [2, 3]
if ndim not in acceptable_ndims:
raise ValueError('Only 2 and 3 dimensional networks are supported')
for i in range(n_upsample):
if ndim == 2:
x = Conv2D(n_filters, (3, 3), strides=(1, 1),
padding='same', data_format='channels_last')(x)
if i == n_upsample - 1 and target is not None:
x = UpsampleLike()([x, target])
else:
x = UpSampling2D(size=(2, 2))(x)
else:
x = Conv3D(n_filters, (3, 3, 3), strides=(1, 1, 1),
padding='same', data_format='channels_last')(x)
if i == n_upsample - 1 and target is not None:
x = UpsampleLike()([x, target])
else:
x = UpSampling3D(size=(2, 2, 2))(x)
if n_upsample == 0:
if ndim == 2:
x = Conv2D(n_filters, (3, 3), strides=(1, 1),
padding='same', data_format='channels_last')(x)
else:
x = Conv3D(n_filters, (3, 3, 3), strides=(1, 1, 1),
padding='same', data_format='channels_last')(x)
if target is not None:
x = UpsampleLike()([x, target])
return x
def MT_CAN_3D(n_frame,
nb_filters1,
nb_filters2,
input_shape,
kernel_size=(3, 3, 3),
dropout_rate1=0.25,
dropout_rate2=0.5,
pool_size=(2, 2, 2),
nb_dense=128):
diff_input = Input(shape=input_shape)
rawf_input = Input(shape=input_shape)
d1 = Conv3D(nb_filters1, kernel_size, padding='same',
activation='tanh')(diff_input)
d2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(d1)
# Appearance Branch
r1 = Conv3D(nb_filters1, kernel_size, padding='same',
activation='tanh')(rawf_input)
r2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(r1)
g1 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r2)
g1 = Attention_mask()(g1)
gated1 = multiply([d2, g1])
d3 = AveragePooling3D(pool_size)(gated1)
d4 = Dropout(dropout_rate1)(d3)
d5 = Conv3D(nb_filters2, kernel_size, padding='same',
activation='tanh')(d4)
d6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(d5)
r3 = AveragePooling3D(pool_size)(r2)
r4 = Dropout(dropout_rate1)(r3)
r5 = Conv3D(nb_filters2, kernel_size, padding='same',
activation='tanh')(r4)
r6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(r5)
g2 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r6)
g2 = Attention_mask()(g2)
gated2 = multiply([d6, g2])
d7 = AveragePooling3D(pool_size)(gated2)
d8 = Dropout(dropout_rate1)(d7)
d9 = Flatten()(d8)
d10_y = Dense(nb_dense, activation='tanh')(d9)
d11_y = Dropout(dropout_rate2)(d10_y)
out_y = Dense(n_frame, name='output_1')(d11_y)
d10_r = Dense(nb_dense, activation='tanh')(d9)
d11_r = Dropout(dropout_rate2)(d10_r)
out_r = Dense(n_frame, name='output_2')(d11_r)
model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r])
return model
def sendec_block1(input_tensor): x1 = Conv3D(filters=32, kernel_size=(1, 3, 3), strides=(1, 2, 2), activation='relu', padding='same', data_format='channels_last')(input_tensor) x = Conv3DTranspose(filters=16, kernel_size=(2, 3, 3), strides=(1, 2, 2), padding='same', data_format='channels_last')(x1) x = concatenate([input_tensor, x], axis=-1) x = BatchNormalization()(x) x = Conv3D(filters=16, kernel_size=(1, 3, 3), strides=(1, 1, 1), activation='relu', padding='same', data_format='channels_last')(x) return x1, x
def vertical_plane_convolution(input, plane_filters, l2):
reduce = Conv3D(plane_filters, 1, kernel_regularizer=l2)(input)
shared_squash = Conv3D(plane_filters, [4, 1, 4], kernel_regularizer=l2)
squash_xz = shared_squash(reduce)
box_xz = spread_axis(squash_xz, plane_filters, [1, 3, 2, 4]) # x, z, y, f -> x, y, z, f
rotate = Permute([2, 1, 3, 4])(reduce)
squash_yz = shared_squash(rotate)
box_yz = spread_axis(squash_yz, plane_filters, [3, 1, 2, 4]) # y, z, x, f -> x, y, z, f
return box_xz, box_yz
def upsample(inp,
factor,
nchannels,
bn=None,
activation=None,
bias=False,
dilation_rate=1,
prefix='unet_3d',
idx=0,
upsampling='copy',
residual=False):
if residual:
resized = UpSampling3D(size=(1, factor, factor))(inp)
resized = Conv3D(nchannels, (1, 1, 1), strides=1,
padding='same')(resized)
resized2 = Conv3DTranspose(nchannels, (1, factor, factor),
strides=(1, factor, factor),
name=prefix + "_deconv3d_" + str(idx),
kernel_initializer='he_normal',
use_bias=bias,
dilation_rate=dilation_rate)(inp)
else:
if upsampling == 'copy':
resized = UpSampling3D(size=(1, factor, factor))(inp)
resized = Conv3D(nchannels, (1, 1, 1), strides=1,
padding='same')(resized)
else:
resized = Conv3DTranspose(nchannels, (1, factor, factor),
strides=(1, factor, factor),
name=prefix + "_deconv3d_" + str(idx),
kernel_initializer='he_normal',
use_bias=bias,
dilation_rate=dilation_rate)(inp)
if bn == 'before':
resized = BatchNormalization(axis=4,
name=prefix + "_batchnorm_" +
str(idx))(resized)
resized = activation(resized)
if bn == 'after':
resized = BatchNormalization(axis=4,
name=prefix + "_batchnorm_" +
str(idx))(resized)
if inp.get_shape().as_list()[-1] == nchannels and residual:
x = inp + x
return resized
def identity_block(input_tensor,
kernel_size,
filters,
stage,
block,
path,
non_degenerate_temporal_conv=False):
"""The identity block is the block that has no conv layer at shortcut.
Arguments:
input_tensor: input tensor
kernel_size: default 3, the kernel size of middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
Returns:
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
if K.image_data_format() == 'channels_last':
bn_axis = 4
else:
bn_axis = 1
conv_name_base = str(path) + 'res' + str(stage) + block + '_branch'
bn_name_base = str(path) + 'bn' + str(stage) + block + '_branch'
if non_degenerate_temporal_conv == True:
x = Conv3D(filters1, (3, 1, 1),
padding='same',
name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
else:
x = Conv3D(filters1, (1, 1, 1),
name=conv_name_base + '2a')(input_tensor)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
x = Activation('relu')(x)
x = Conv3D(filters2,
kernel_size,
padding='same',
name=conv_name_base + '2b')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
x = Activation('relu')(x)
x = Conv3D(filters3, (1, 1, 1), name=conv_name_base + '2c')(x)
x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
x = layers.add([x, input_tensor])
x = Activation('relu')(x)
return x
def default_regression_model(num_values,
num_anchors,
pyramid_feature_size=256,
regression_feature_size=256,
name='regression_submodel'):
"""Creates the default regression submodel.
Args:
num_values: Number of values to regress.
num_anchors: Number of anchors to regress for each feature level.
pyramid_feature_size: The number of filters to expect from the
feature pyramid levels.
regression_feature_size: The number of filters to use in the layers
in the regression submodel.
name: The name of the submodel.
Returns:
A keras.models.Model that predicts regression values for each anchor.
"""
# All new conv layers except the final one in the
# RetinaNet (classification) subnets are initialized
# with bias b = 0 and a Gaussian weight fill with stddev = 0.01.
options = {
'kernel_size': 3,
'strides': 1,
'padding': 'same',
'kernel_initializer': RandomNormal(mean=0.0, stddev=0.01, seed=None),
'bias_initializer': 'zeros'
}
if K.image_data_format() == 'channels_first':
inputs = Input(shape=(pyramid_feature_size, None, None, None))
else:
inputs = Input(shape=(None, None, None, pyramid_feature_size))
outputs = inputs
for i in range(4):
outputs = Conv3D(filters=regression_feature_size,
activation='relu',
name='pyramid_regression_{}'.format(i),
**options)(outputs)
outputs = Conv3D(num_anchors * num_values,
name='pyramid_regression',
**options)(outputs)
if K.image_data_format() == 'channels_first':
outputs = Permute((2, 3, 1),
name='pyramid_regression_permute')(outputs)
outputs = Reshape((-1, num_values),
name='pyramid_regression_reshape')(outputs)
return Model(inputs=inputs, outputs=outputs, name=name)
def modelForward(self, in_data, is_train=False):
in_cen, in_per = in_data
def probTrain():
return self.__do_prob
def probValid():
return 1.0
do_prob = tf.cond(tf.equal(is_train, tf.constant(True, dtype=tf.bool)),
probTrain, probValid)
with tf.variable_scope('model_scope', reuse=tf.AUTO_REUSE) as scope:
## 1 conv -> batch norm -> activate
conv1 = Conv3D(self.__filters[0],
kernel_size=self.__kernels[0],
strides=self.__strides[0],
padding=self.__paddings[0])(in_cen)
conv1 = BatchNormalization(momentum=0.9)(conv1, is_train)
conv1 = LeakyReLU()(conv1)
## pad act1 and then add it to in_per (to the peripheral modules)
cen_paddings = tf.constant([[0, 0], [1, 1], [1, 1], [0, 0],
[0, 0]]) # pad only the cross-section
conv1_padded = tf.pad(conv1, cen_paddings)
conv1_combined = tf.add(conv1_padded, in_per)
## 2 conv -> batch norm -> activate
conv2 = Conv3D(self.__filters[1],
kernel_size=self.__kernels[1],
strides=self.__strides[1],
padding=self.__paddings[1])(conv1_combined)
bn2 = BatchNormalization(momentum=0.9)(conv2, is_train)
act2 = LeakyReLU()(bn2)
# Flatten
flatten = Flatten()(act2)
## 1 FC
fc1 = Dense(self.__fc_dim)(flatten)
fc1 = LeakyReLU()(fc1)
## Dropout
fc1_drop = Dropout(do_prob)(fc1)
## 2 FC
logits = Dense(2)(fc1_drop)
return logits
def __init__(self,
original_dim,
intermediate_dim=64,
name="image",
**kwargs):
super(OSIC_Image, self).__init__(name=name, **kwargs)
self.layers = []
self.layers.append(InputLayer(input_shape=original_dim))
self.layers.append(
Conv3D(filters=8,
kernel_size=5,
strides=3,
padding="same",
kernel_initializer=GlorotUniform(seed=0),
input_shape=original_dim))
self.layers.append(LayerNormalization())
self.layers.append(Activation('elu'))
self.layers.append(
Conv3D(filters=16,
kernel_size=2,
strides=2,
padding="same",
kernel_initializer=GlorotUniform(seed=0)))
self.layers.append(LayerNormalization())
self.layers.append(Activation('elu'))
self.layers.append(
Conv3D(filters=32,
kernel_size=2,
strides=1,
padding="same",
kernel_initializer=GlorotUniform(seed=0)))
self.layers.append(LayerNormalization())
self.layers.append(Activation('elu'))
self.layers.append(
Conv3D(filters=64,
kernel_size=2,
strides=1,
padding="same",
kernel_initializer=GlorotUniform(seed=0)))
self.layers.append(LayerNormalization())
self.layers.append(Activation('elu'))
# self.layers.append(Conv3DTranspose(32, 2, 1))
# self.layers.append(LayerNormalization())
# self.layers.append(Activation('elu'))
# self.layers.append(Conv3DTranspose(16, 2, 1))
# self.layers.append(LayerNormalization())
#self.layers.append(Conv3D(filters=1,kernel_size=5,strides=4,kernel_initializer=GlorotUniform(seed=0)))
# self.layers.append(Conv3D(filters=2,kernel_size=1,activation="softmax", kernel_initializer=GlorotUniform(seed=0)))
self.layers.append(Dense(64))
self.layers.append(LayerNormalization())
def compute_feature_extractor(feature, shape):
if feature == 'appearance':
# This should not stay: channels_first/last should be used to
# dictate size (1 works for either right now)
N_layers = np.int(np.floor(np.log2(input_shape[1])))
feature_extractor = Sequential()
feature_extractor.add(InputLayer(input_shape=shape))
# feature_extractor.add(ImageNormalization2D('std', filter_size=32))
for layer in range(N_layers):
feature_extractor.add(
Conv3D(64, (1, 3, 3),
kernel_initializer=init,
padding='same',
kernel_regularizer=l2(reg)))
feature_extractor.add(BatchNormalization(axis=channel_axis))
feature_extractor.add(Activation('relu'))
feature_extractor.add(MaxPool3D(pool_size=(1, 2, 2)))
feature_extractor.add(Reshape((-1, 64)))
return feature_extractor
elif feature == 'distance':
return None
elif feature == 'neighborhood':
N_layers_og = np.int(
np.floor(np.log2(2 * neighborhood_scale_size + 1)))
feature_extractor_neighborhood = Sequential()
feature_extractor_neighborhood.add(
InputLayer(input_shape=(None, 2 * neighborhood_scale_size + 1,
2 * neighborhood_scale_size + 1, 1)))
for layer in range(N_layers_og):
feature_extractor_neighborhood.add(
Conv3D(64, (1, 3, 3),
kernel_initializer=init,
padding='same',
kernel_regularizer=l2(reg)))
feature_extractor_neighborhood.add(
BatchNormalization(axis=channel_axis))
feature_extractor_neighborhood.add(Activation('relu'))
feature_extractor_neighborhood.add(
MaxPool3D(pool_size=(1, 2, 2)))
feature_extractor_neighborhood.add(Reshape((-1, 64)))
return feature_extractor_neighborhood
elif feature == 'regionprop':
return None
else:
raise ValueError('siamese_model.compute_feature_extractor: '
'Unknown feature `{}`'.format(feature))
def channel_attention_m(x, residual=False, stream=False):
if not stream:
# dims: BxHxWxCxM (M streams)
if isinstance(x, list):
x = Lambda(lambda var: K.stack(var, axis=4))(x)
y = GlobalMaxPooling3D()(x)
y = Lambda(lambda var: K.expand_dims(
K.expand_dims(K.expand_dims(var, axis=1), axis=2), axis=3))(y)
y = Conv3D(filters=int(K.int_shape(x)[-1] / 2),
kernel_size=1,
strides=1)(y)
y = Activation("relu")(y)
y = Conv3D(filters=K.int_shape(x)[-1], kernel_size=1, strides=1)(y)
y = Activation("softmax")(y)
y = Lambda(lambda var: tf.multiply(*var))([x, y])
if residual:
y = Add()([y, x])
else:
# dims: BxHxWxCxM (M streams)
y = GlobalMaxPooling3D()(x)
y = Lambda(lambda var: K.expand_dims(
K.expand_dims(K.expand_dims(var, axis=1), axis=2), axis=3))(y)
y = Conv3D(filters=int(K.int_shape(x)[-1] / 2),
kernel_size=1,
strides=1)(y)
y = Activation("relu")(y)
y = Conv3D(filters=2, kernel_size=1, strides=1)(y)
y = Activation("sigmoid")(y)
y_l = []
c = int(x.get_shape().as_list()[-1] / 2)
for i in range(2):
ind_st = i * c
ind_end = (i + 1) * c
x_sub = Lambda(slicing,
arguments={
'index': ind_st,
'index_end': ind_end
})(x)
y_sub = Lambda(slicing, arguments={
'index': i,
'index_end': i + 1
})(y)
y = Lambda(lambda var: tf.multiply(*var))([x_sub, y_sub])
if residual:
y = Add()([y, x_sub])
y_l.append(y)
y = concatenate(y_l)
return y
def _shortcut(input, residual):
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width, height)
# Should be int if network architecture is correctly configured.
input_shape = K.int_shape(input)
residual_shape = K.int_shape(residual)
stride_width = int(
round(input_shape[CONV_DIM1] / residual_shape[CONV_DIM1]))
stride_height = int(
round(input_shape[CONV_DIM2] / residual_shape[CONV_DIM2]))
stride_depth = int(
round(input_shape[CONV_DIM3] / residual_shape[CONV_DIM3]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
# 1 X 1 conv if shape is different. Else identity.
if stride_width > 1 or stride_height > 1 or not equal_channels:
shortcut = Conv3D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1, 1),
strides=(stride_width, stride_height, stride_depth),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(0.0001))(input)
return add([shortcut, residual])
def __merge_temporal_features(feature,
mode='conv',
feature_size=256,
frames_per_batch=1):
if mode == 'conv':
x = Conv3D(
feature_size,
(frames_per_batch, 3, 3),
strides=(1, 1, 1),
padding='same',
)(feature)
x = BatchNormalization(axis=-1)(x)
x = Activation('relu')(x)
elif mode == 'lstm':
x = ConvLSTM2D(feature_size, (3, 3),
padding='same',
activation='relu',
return_sequences=True)(feature)
elif mode == 'gru':
x = ConvGRU2D(feature_size, (3, 3),
padding='same',
activation='relu',
return_sequences=True)(feature)
temporal_feature = x
return temporal_feature
