def upsample(inp,
factor,
nchannels,
bn=None,
activation=None,
bias=False,
dilation_rate=1,
prefix='unet_3d',
idx=0,
upsampling='copy',
residual=False):
if residual:
resized = UpSampling3D(size=(1, factor, factor))(inp)
resized = Conv3D(nchannels, (1, 1, 1), strides=1,
padding='same')(resized)
resized2 = Conv3DTranspose(nchannels, (1, factor, factor),
strides=(1, factor, factor),
name=prefix + "_deconv3d_" + str(idx),
kernel_initializer='he_normal',
use_bias=bias,
dilation_rate=dilation_rate)(inp)
else:
if upsampling == 'copy':
resized = UpSampling3D(size=(1, factor, factor))(inp)
resized = Conv3D(nchannels, (1, 1, 1), strides=1,
padding='same')(resized)
else:
resized = Conv3DTranspose(nchannels, (1, factor, factor),
strides=(1, factor, factor),
name=prefix + "_deconv3d_" + str(idx),
kernel_initializer='he_normal',
use_bias=bias,
dilation_rate=dilation_rate)(inp)
if bn == 'before':
resized = BatchNormalization(axis=4,
name=prefix + "_batchnorm_" +
str(idx))(resized)
resized = activation(resized)
if bn == 'after':
resized = BatchNormalization(axis=4,
name=prefix + "_batchnorm_" +
str(idx))(resized)
if inp.get_shape().as_list()[-1] == nchannels and residual:
x = inp + x
return resized
def upsampling_module(layer, filters):
upsample = UpSampling3D(size=(2, 2, 2),
data_format='channels_first')(layer)
upsample_conv = convolution_module(upsample,
filters,
activation=LeakyReLU,
instance_norm=True)
return upsample_conv
def semantic_upsample(x, n_upsample, n_filters=64, ndim=2, target=None):
"""
Performs iterative rounds of 2x upsampling and
convolutions with a 3x3 filter to remove aliasing effects
Args:
x (tensor): The input tensor to be upsampled
n_upsample (int): The number of 2x upsamplings
n_filters (int, optional): Defaults to 256. The number of filters for
the 3x3 convolution
target (tensor, optional): Defaults to None. A tensor with the target
shape. If included, then the final upsampling layer will reshape
to the target tensor's size
ndim: The spatial dimensions of the input data.
Default is 2, but it also works with 3
Returns:
The upsampled tensor
"""
acceptable_ndims = [2, 3]
if ndim not in acceptable_ndims:
raise ValueError('Only 2 and 3 dimensional networks are supported')
for i in range(n_upsample):
if ndim == 2:
x = Conv2D(n_filters, (3, 3), strides=(1, 1),
padding='same', data_format='channels_last')(x)
if i == n_upsample - 1 and target is not None:
x = UpsampleLike()([x, target])
else:
x = UpSampling2D(size=(2, 2))(x)
else:
x = Conv3D(n_filters, (3, 3, 3), strides=(1, 1, 1),
padding='same', data_format='channels_last')(x)
if i == n_upsample - 1 and target is not None:
x = UpsampleLike()([x, target])
else:
x = UpSampling3D(size=(2, 2, 2))(x)
if n_upsample == 0:
if ndim == 2:
x = Conv2D(n_filters, (3, 3), strides=(1, 1),
padding='same', data_format='channels_last')(x)
else:
x = Conv3D(n_filters, (3, 3, 3), strides=(1, 1, 1),
padding='same', data_format='channels_last')(x)
if target is not None:
x = UpsampleLike()([x, target])
return x
def decoder_block_3(x,
nr_of_convolutions,
cross_over_connection=None,
use_bn=False,
spatial_dropout=None):
x = UpSampling3D((2, 2, 2))(x)
if cross_over_connection is not None:
x = Concatenate()([cross_over_connection, x])
x = convolution_block_3(x, nr_of_convolutions, use_bn, spatial_dropout)
return x
def UNet3D_Isensee(input_shape=(4, 128, 128, 128),
nchannels=(16, 32, 64, 128, 256),
n_labels=3,
activation='sigmoid',
n_segmentation_level=3):
inputs = Input(input_shape)
current_layer = inputs
downsample_level = []
segmentation_level = []
for idx, filters in enumerate(nchannels):
if idx == 0:
in_conv = convolution_module(current_layer,
filters,
activation=LeakyReLU,
instance_norm=True)
else:
in_conv = convolution_module(current_layer,
filters,
activation=LeakyReLU,
strides=(2, 2, 2),
instance_norm=True)
context_layer = context_module(in_conv, filters, dropout=0.3)
add_layer = Add()([in_conv, context_layer])
downsample_level.append(add_layer)
current_layer = add_layer
for idx, filters in reversed(list(enumerate(nchannels[:-1:]))):
upsample_conv = upsampling_module(current_layer, filters)
current_layer = concatenate([upsample_conv, downsample_level[idx]],
axis=1)
localization = localization_module(current_layer, filters)
current_layer = localization
if idx < n_segmentation_level:
layer = Conv3D(n_labels,
kernel_size=(1, 1, 1),
data_format='channels_first')(localization)
segmentation_level.append(layer)
output_layer = None
for idx in range(len(segmentation_level)):
if output_layer is None:
output_layer = segmentation_level[idx]
else:
output_layer = Add()([output_layer, segmentation_level[idx]])
if idx != len(segmentation_level) - 1:
output_layer = UpSampling3D(
size=(2, 2, 2), data_format='channels_first')(output_layer)
act = Activation(activation)(output_layer)
model = Model(inputs=inputs, outputs=act)
return model
def test_delete_channels_upsampling3d(channel_index, data_format):
layer = UpSampling3D([2, 3, 2], data_format=data_format)
layer_test_helper_flatten_3d(layer, channel_index, data_format=data_format)
def get_prediction_model(name, in_shape, include_top, algorithm_instance, num_classes, kwargs):
if name == "big_fully":
input_l = Input(in_shape)
output_l = fully_connected_big(input_l, include_top=include_top, **kwargs)
model = Model(input_l, output_l)
elif name == "simple_multiclass":
input_l = Input(in_shape)
output_l = simple_multiclass(input_l, include_top=include_top, **kwargs)
model = Model(input_l, output_l)
elif name == "unet_2d_upconv":
assert algorithm_instance is not None, "no algorithm instance for 2d skip connections found"
assert algorithm_instance.layer_data is not None, "no layer data for 2d skip connections found"
first_input = Input(in_shape)
includes_pooling = algorithm_instance.layer_data[2]
if includes_pooling:
x = UpSampling2D((2, 2))(first_input)
else:
x = first_input
inputs_skip = [Input(x.shape[1:]) for x in reversed(algorithm_instance.layer_data[0])]
inputs_up = [x] + inputs_skip
model_up_out = upconv_model(x.shape[1:], down_layers=algorithm_instance.layer_data[0],
filters=algorithm_instance.layer_data[1], num_classes=num_classes)(inputs_up)
return Model(inputs=[first_input, *inputs_skip], outputs=model_up_out)
elif name == "unet_3d_upconv":
assert algorithm_instance is not None, "no algorithm instance for 3d skip connections found"
assert algorithm_instance.layer_data is not None, "no layer data for 3d skip connections found"
first_input = Input(in_shape)
includes_pooling = algorithm_instance.layer_data[2]
if includes_pooling:
x = UpSampling3D((2, 2, 2))(first_input)
else:
x = first_input
inputs_skip = [Input(x.shape[1:]) for x in reversed(algorithm_instance.layer_data[0])]
inputs_up = [x] + inputs_skip
model_up_out = upconv_model_3d(x.shape[1:], down_layers=algorithm_instance.layer_data[0],
filters=algorithm_instance.layer_data[1], num_classes=num_classes)(inputs_up)
return Model(inputs=[first_input, *inputs_skip], outputs=model_up_out)
elif name == "unet_3d_upconv_patches":
# This version of the unet3d model creates a separate unet for each patch. Currently unused
assert algorithm_instance is not None, "no algorithm instance for 3d skip connections found"
assert algorithm_instance.layer_data is not None, "no layer data for 3d skip connections found"
n_patches = in_shape[0]
embed_dim = in_shape[1]
# combine all predictions from encoders to one layer and split up again
first_input = Input(in_shape)
flat = Flatten()(first_input)
processed_first_input = Dense(n_patches * embed_dim, activation="relu")(flat)
processed_first_input = Reshape((n_patches, embed_dim))(processed_first_input)
# get the first shape of the upconv from the encoder
# get whether the last layer is a pooling layer
first_l_shape = algorithm_instance.layer_data[2][0]
includes_pooling = algorithm_instance.layer_data[2][1]
units = np.prod(first_l_shape)
# build small model that selects a small shape from the unified predictions
model_first_up = Sequential()
model_first_up.add(Input(embed_dim))
model_first_up.add(Dense(units, activation="relu"))
model_first_up.add(Reshape(first_l_shape))
if includes_pooling:
model_first_up.add(UpSampling3D((2, 2, 2)))
# apply selection to get input for decoder models
processed_first_input = TimeDistributed(model_first_up)(processed_first_input)
# prepare decoder
model_up = upconv_model_3d(processed_first_input.shape[2:], down_layers=algorithm_instance.layer_data[0],
filters=algorithm_instance.layer_data[1], num_classes=num_classes)
pred_patches = []
large_inputs = [first_input]
for s in reversed(algorithm_instance.layer_data[0]):
large_inputs.append(Input(n_patches + s.shape[1:]))
for p in range(n_patches):
y = [Lambda(lambda x: x[:, p, :, :, :, :], output_shape=processed_first_input.shape[2:])]
for s in reversed(algorithm_instance.layer_data[0]):
y.append(Lambda(lambda x: x[:, p, :, :, :, :], output_shape=s.shape[1:]))
small_inputs = [y[0](processed_first_input)] # the first input has to be processed
for i in range(1, len(large_inputs)):
small_inputs.append(y[i](large_inputs[i])) # we can take the rest as is
pred_patches.append(model_up(small_inputs))
last_out = Concatenate(axis=1)(pred_patches)
last_out = Reshape((n_patches,) + model_up.layers[-1].output_shape[1:])(last_out)
model = Model(inputs=large_inputs, outputs=[last_out])
elif name == "none":
return None
else:
raise ValueError("model " + name + " not found")
return model
def manet(input_layer_names, input_shape, config=None, logger=None):
output_layers = []
# paras of multi-scales
nb_pyr_levels = 3
pyr_levels = list(range(nb_pyr_levels))
ret_feat_levels = 2
# paras of layers
conv_type, ks, activ = "conv", 2, "relu"
# paras of cost volume (cv)
min_disp, max_disp, num_disp_labels = -4, 4, 80
########## input layers ##########
input_layers = []
for _, input_layer_name in enumerate(input_layer_names):
x = Input(shape=input_shape, name=input_layer_name)
input_layers.append(x)
########## Branch_2, 3: cv ##########
pyr_outputs = [] # outputs of pyramid level 1, 2
# 1. Feature extraction
nb_filt1 = 8
feature_s_paras = {
'ks': ks,
'stride': [2, 1],
'padding': "zero",
'filter': [nb_filt1, nb_filt1 * 2] * 1,
'activation': activ,
'conv_type': conv_type,
'pyr': True,
'layer_nums': 2,
"ret_feat_levels": ret_feat_levels
}
feature_s_m = feature_extraction_m((input_shape[0], input_shape[1], 1),
feat_paras=feature_s_paras)
fs_ts_ids = []
feature_streams = []
for stream_id, x in enumerate(input_layers):
if stream_id > 1:
continue
feature_stream = []
for x_sid in range(input_shape[2]):
x_sub = Lambda(slicing, arguments={'index': x_sid})(x)
x_sub = feature_s_m(x_sub)
feature_stream.append(x_sub)
if stream_id == 0:
t_ids = list(range(input_shape[2]))[::-1]
s_ids = [int((input_shape[2] - 1) / 2)] * input_shape[2]
elif stream_id == 1:
t_ids = [int((input_shape[2] - 1) / 2)] * input_shape[2]
s_ids = list(range(input_shape[2]))
fs_ts_ids.append((t_ids, s_ids))
feature_streams.append(feature_stream)
# 2/3/4. Cost volume + 3D aggregation + Regression
cv_ca_pyr_levels = pyr_levels[1:]
for pyr_level in cv_ca_pyr_levels[::-1]:
cv_streams = []
scale_factor = math.pow(2, pyr_level)
pyr_level_ndl = int(num_disp_labels / scale_factor)
# 2. Cost volume
for fs_id, feature_stream in enumerate(feature_streams):
pyr_fs = [fs_ep[pyr_level - 1] for fs_ep in feature_stream]
cost_volume = Lambda(compute_cost_volume,
arguments={
"t_s_ids": fs_ts_ids[fs_id],
"min_disp": min_disp / scale_factor,
"max_disp": max_disp / scale_factor,
"labels": pyr_level_ndl,
"move_path": "LT"
})(pyr_fs)
cv_streams.append(cost_volume)
# Multiple streams
if len(cv_streams) > 1:
cost_volume = concatenate(cv_streams)
# 3/4. 3D aggregation + Regression
# 3. 3D aggregation
if pyr_level == cv_ca_pyr_levels[0]:
ca_paras = {
'ks': 3,
'stride': 2,
'padding': "same",
'filter': nb_filt1 * 2,
'activation': activ,
'conv_type': conv_type,
'n_dc': 1
}
output = cost_aggregation(cost_volume, ca_paras=ca_paras)
else:
ca_paras = {
'ks': 3,
'stride': 2,
'padding': "same",
'filter': nb_filt1 * 4,
'activation': activ,
'conv_type': conv_type,
'n_dc': 1
}
output = cost_aggregation(cost_volume, ca_paras=ca_paras)
output = UpSampling3D(size=(2, 2, 2),
name="u_s{}".format(pyr_level))(output)
# 4. Regression
logger.info("=> regression at scale level {}".format(pyr_level))
output = Lambda(lambda op: tf.nn.softmax(op, axis=1))(output)
pl_o = Lambda(soft_min_reg,
arguments={
"axis": 1,
"min_disp": min_disp,
"max_disp": max_disp,
"labels": num_disp_labels
},
name="sm_disp{}".format(pyr_level))(output)
pyr_outputs.append(pl_o)
d2 = Average()(pyr_outputs[:2]) # outputs at scale level 1 and 2
########## Branch_1: no_cv ##########
block_n = 8 # blocks
ifn = 40 # filter
# Branch_1: 2D aggregation
pl_features = []
pl_feature_streams = []
for x in input_layers:
x = ReflectionPadding2D(padding=([4, 4], [4, 4]))(x)
feature_paras = {
'ks': ks,
'stride': 1,
'padding': "zero",
'filter': 1 * [ifn],
'activation': activ,
'conv_type': conv_type,
'layer_nums': 1
}
x = cna_m(x, feature_paras, layer_names='random')
pl_feature_streams.append(x)
x = concatenate(pl_feature_streams) # merge layers
pl_features.append(x)
pyr_level = pyr_levels[0] # = 0
fn = [i for i in block_n * [ifn * len(input_layer_names)]]
cna_paras = {
'ks': ks,
'stride': 1,
'padding': "valid",
'filter': fn,
'activation': activ,
'conv_type': conv_type,
'layer_nums': block_n
}
x = cna_m(pl_features[pyr_level], cna_paras, layer_names='random')
x = conv_2d(x, num_disp_labels, ks=ks, padding="zero")
# Branch_1: Regression
logger.info("=> regression at scale level {}".format(pyr_level))
x = Lambda(lambda op: tf.nn.softmax(op, axis=-1))(x)
d1 = Lambda(soft_min_reg,
arguments={
"axis": -1,
"min_disp": min_disp,
"max_disp": max_disp,
"labels": num_disp_labels
},
name="sm_disp_{}".format(pyr_level))(x)
########## Output ##########
d0 = Average()([d2, d1])
output_layers.append(d2)
output_layers.append(d1)
output_layers.append(d0)
manet_model = Model(inputs=input_layers, outputs=output_layers)
if config.model_infovis:
manet_model.summary()
return manet_model
def UNet_3D(input_shape=(None, None, None, 1),
depth=5,
max_filters=256,
kernel_size=(3, 3, 3),
n_output_channels=1,
activation='sigmoid',
drop_out=0,
dropout_at_prediction=False,
batch_norm=False):
'''U net architecture (down/up sampling with skip architecture)
See: http://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/
Keras implementatio:; https://github.com/jocicmarko/ultrasound-nerve-segmentation/blob/master/train.py
'''
def Conv3DReluBatchNorm(n_filters,
kernel_size,
strides,
inputs,
drop_out,
batch_norm,
name=None):
c = Conv3D(n_filters,
kernel_size,
strides=strides,
padding='same',
kernel_initializer='glorot_normal',
activation='relu')(inputs)
if batch_norm:
c = BatchNormalization(scale=False)(c)
if drop_out:
if dropout_at_prediction:
c = Dropout(rate=drop_out)(c, training=True)
else:
c = Dropout(rate=drop_out)(c)
return c
if max_filters % 2 != 0:
raise Exception('max_filters must be divisible by 2!')
layer_stack = []
inputs = Input(input_shape)
layer = Conv3DReluBatchNorm(int(max_filters / 2**depth), kernel_size,
(1, 1, 1), inputs, drop_out, batch_norm,
'conv3D_0')
layer_stack.append(layer)
for i in range(1, depth + 1):
layer = Conv3DReluBatchNorm(int(max_filters / 2**(depth - i)),
kernel_size, (2, 2, 2), layer, drop_out,
batch_norm, 'conv3D_' + str(i))
layer_stack.append(layer)
layer_stack.pop()
for i in range(depth - 1, -1, -1):
layer = Concatenate(axis=-1)(
[UpSampling3D(size=(2, 2, 2))(layer),
layer_stack.pop()])
layer = Conv3DReluBatchNorm(int(max_filters / 2**(depth - i)),
kernel_size, (1, 1, 1), layer, drop_out,
batch_norm,
'conv3D_' + str(depth + (depth - i + 1)))
outputs = Conv3D(n_output_channels, (1, 1, 1),
strides=(1, 1, 1),
activation=activation)(layer)
model = Model(inputs=inputs, outputs=outputs)
return model