def seg_pnn(feature_num=128,
timesteps=256,
multi_grid_layer_n=5,
multi_grid_n=3,
prev_model="melody_transfer_transpose"):
layer_out = []
input_score_48 = Input(shape=(timesteps, feature_num, 1),
name="input_score_48")
input_score_12 = Input(shape=(timesteps, feature_num // 3, 1),
name="input_score_12")
me_transfer_seg = seg(multi_grid_layer_n=1, timesteps=timesteps, prog=True)
me_seg = load_model(prev_model)
model_copy(me_seg, me_transfer_seg)
#TODO: move inside model_copy
for index, layer in enumerate(me_transfer_seg.layers):
me_transfer_seg.layers[index].trainable = False
o_p = me_transfer_seg([input_score_12])
en_l = Conv2D(2**5, (7, 7), strides=(1, 1), padding="same")(input_score_48)
o = adapter(o_p[0], 2**(5), dropout_rate=0.2)
en_l = add([en_l, o])
en_l1 = conv_block(en_l, 2**5, (3, 3), strides=(2, 2))
en_l1 = conv_block(en_l1, 2**5, (3, 3), strides=(1, 1))
layer_out.append(en_l1)
o = adapter(o_p[1], 2**(5), dropout_rate=0.2)
en_l1 = add([en_l1, o])
en_l2 = conv_block(en_l1, 2**6, (3, 3), strides=(2, 2))
en_l2 = conv_block(en_l2, 2**6, (3, 3), strides=(1, 1))
en_l2 = conv_block(en_l2, 2**6, (3, 3), strides=(1, 1))
layer_out.append(en_l2)
o = adapter(o_p[2], 2**(6), dropout_rate=0.2)
en_l2 = add([en_l2, o])
en_l3 = conv_block(en_l2, 2**7, (3, 3), strides=(2, 2))
en_l3 = conv_block(en_l3, 2**7, (3, 3), strides=(1, 1))
en_l3 = conv_block(en_l3, 2**7, (3, 3), strides=(1, 1))
en_l3 = conv_block(en_l3, 2**7, (3, 3), strides=(1, 1))
layer_out.append(en_l3)
o = adapter(o_p[3], 2**(7), dropout_rate=0.2)
en_l3 = add([en_l3, o])
en_l4 = conv_block(en_l3, 2**8, (3, 3), strides=(2, 2))
en_l4 = conv_block(en_l4, 2**8, (3, 3), strides=(1, 1))
en_l4 = conv_block(en_l4, 2**8, (3, 3), strides=(1, 1))
en_l4 = conv_block(en_l4, 2**8, (3, 3), strides=(1, 1))
en_l4 = conv_block(en_l4, 2**8, (3, 3), strides=(1, 1))
layer_out.append(en_l4)
o = adapter(o_p[4], 2**(8), dropout_rate=0.2)
en_l4 = add([en_l4, o])
feature = en_l4
for i in range(multi_grid_layer_n):
feature = BatchNormalization()(Activation("relu")(feature))
feature = Dropout(0.3)(feature)
m = BatchNormalization()(Conv2D(2**9, (1, 1),
strides=(1, 1),
padding="same",
activation="relu")(feature))
multi_grid = m
for ii in range(multi_grid_n):
m = BatchNormalization()(Conv2D(2**9, (3, 3),
strides=(1, 1),
dilation_rate=2**ii,
padding="same",
activation="relu")(feature))
multi_grid = concatenate([multi_grid, m])
multi_grid = Dropout(0.3)(multi_grid)
feature = Conv2D(2**9, (1, 1), strides=(1, 1),
padding="same")(multi_grid)
o = adapter(o_p[5], 2**(9), dropout_rate=0.3)
feature = add([feature, o])
feature = BatchNormalization()(Activation("relu")(feature))
feature = Dropout(0.4)(feature)
feature = Conv2D(2**8, (1, 1), strides=(1, 1), padding="same")(feature)
feature = add([feature, layer_out[3]])
de_l1 = transpose_conv_block(feature, 2**7, (3, 3), strides=(2, 2))
o = adapter(o_p[6], 2**(7), kernel_size=(1, 5), dropout_rate=0.4)
de_l1 = add([de_l1, o])
skip = de_l1
de_l1 = BatchNormalization()(Activation("relu")(de_l1))
de_l1 = concatenate(
[de_l1, BatchNormalization()(Activation("relu")(layer_out[2]))])
de_l1 = Dropout(0.4)(de_l1)
de_l1 = Conv2D(2**7, (1, 1), strides=(1, 1), padding="same")(de_l1)
de_l1 = add([de_l1, skip])
de_l2 = transpose_conv_block(de_l1, 2**6, (3, 3), strides=(2, 2))
o = adapter(o_p[7], 2**(6), kernel_size=(1, 5), dropout_rate=0.4)
de_l2 = add([de_l2, o])
skip = de_l2
de_l2 = BatchNormalization()(Activation("relu")(de_l2))
de_l2 = concatenate(
[de_l2, BatchNormalization()(Activation("relu")(layer_out[1]))])
de_l2 = Dropout(0.4)(de_l2)
de_l2 = Conv2D(2**6, (1, 1), strides=(1, 1), padding="same")(de_l2)
de_l2 = add([de_l2, skip])
de_l3 = transpose_conv_block(de_l2, 2**5, (3, 3), strides=(2, 2))
o = adapter(o_p[8], 2**(5), kernel_size=(1, 5), dropout_rate=0.4)
de_l3 = add([de_l3, o])
skip = de_l3
de_l3 = BatchNormalization()(Activation("relu")(de_l3))
de_l3 = concatenate(
[de_l3, BatchNormalization()(Activation("relu")(layer_out[0]))])
de_l3 = Dropout(0.4)(de_l3)
de_l3 = Conv2D(2**5, (1, 1), strides=(1, 1), padding="same")(de_l3)
de_l3 = add([de_l3, skip])
de_l4 = transpose_conv_block(de_l3, 2**5, (3, 3), strides=(2, 2))
o = adapter(o_p[9], 2**(5), kernel_size=(1, 5), dropout_rate=0.4)
de_l4 = add([de_l4, o])
de_l4 = BatchNormalization()(Activation("relu")(de_l4))
de_l4 = Dropout(0.4)(de_l4)
out = Conv2D(2, (1, 1), strides=(1, 1), padding="same",
name='prediction')(de_l4)
model = Model(inputs=[input_score_48, input_score_12], outputs=out)
return model
def unet_model_3d(input_shape, pool_size=(2, 2, 2), n_labels=3, initial_learning_rate=0.00001, deconvolution=False,
depth=4, n_base_filters=32, include_label_wise_dice_coefficients=False, metrics=dice_coefficient,
batch_normalization=False, activation_name="sigmoid", loss_fn=weighted_dice_coefficient_loss, multi_gpu=False):
"""
Builds the 3D UNet Keras model.f
:param metrics: List metrics to be calculated during model training (default is dice coefficient).
:param include_label_wise_dice_coefficients: If True and n_labels is greater than 1, model will report the dice
coefficient for each label as metric.
:param n_base_filters: The number of filters that the first layer in the convolution network will have. Following
layers will contain a multiple of this number. Lowering this number will likely reduce the amount of memory required
to train the model.
:param depth: indicates the depth of the U-shape for the model. The greater the depth, the more max pooling
layers will be added to the model. Lowering the depth may reduce the amount of memory required for training.
:param input_shape: Shape of the input data (n_chanels, x_size, y_size, z_size). The x, y, and z sizes must be
divisible by the pool size to the power of the depth of the UNet, that is pool_size^depth.
:param pool_size: Pool size for the max pooling operations.
:param n_labels: Number of binary labels that the model is learning.
:param initial_learning_rate: Initial learning rate for the model. This will be decayed during training.
:param deconvolution: If set to True, will use transpose convolution(deconvolution) instead of up-sampling. This
increases the amount memory required during training.
:return: Untrained 3D UNet Model
"""
inputs = Input(input_shape)
current_layer = inputs
levels = list()
# add levels with max pooling
for layer_depth in range(depth):
layer1 = create_convolution_block(input_layer=current_layer, n_filters=n_base_filters*(2**layer_depth),
batch_normalization=batch_normalization)
layer2 = create_convolution_block(input_layer=layer1, n_filters=n_base_filters*(2**layer_depth)*2,
batch_normalization=batch_normalization)
if layer_depth < depth - 1:
current_layer = MaxPooling3D(pool_size=pool_size)(layer2)
levels.append([layer1, layer2, current_layer])
else:
current_layer = layer2
levels.append([layer1, layer2])
# add levels with up-convolution or up-sampling
for layer_depth in range(depth-2, -1, -1):
up_convolution = get_up_convolution(pool_size=pool_size, deconvolution=deconvolution,
n_filters=current_layer._keras_shape[1])(current_layer)
concat = concatenate([up_convolution, levels[layer_depth][1]], axis=1)
current_layer = create_convolution_block(n_filters=levels[layer_depth][1]._keras_shape[1],
input_layer=concat, batch_normalization=batch_normalization)
current_layer = create_convolution_block(n_filters=levels[layer_depth][1]._keras_shape[1],
input_layer=current_layer,
batch_normalization=batch_normalization)
final_convolution = Conv3D(n_labels, (1, 1, 1))(current_layer)
act = Activation(activation_name)(final_convolution)
model = Model(inputs=inputs, outputs=act)
if not isinstance(metrics, list):
metrics = [metrics]
if include_label_wise_dice_coefficients and n_labels > 1:
lab_names = {
0: 'Necrotic',
1: 'Edema',
2: 'Enhancing'
}
label_wise_dice_metrics = [get_label_dice_coefficient_function(index, name) for index, name in lab_names.iteritems()]
if metrics:
metrics = metrics + label_wise_dice_metrics
else:
metrics = label_wise_dice_metrics
if multi_gpu == True:
logger.warn('Compiling a MULTI GPU MODEL')
model = multi_gpu_model(model, gpus=4)
logger.warn('Done compiling!')
model.compile(optimizer=Adam(lr=initial_learning_rate), loss=loss_fn, metrics=metrics)
return model
def build_model(self):
""" A basic implementation of the U-Net proposed in https://arxiv.org/abs/1505.04597
TODO: specify optimizer
Returns
-------
Keras model or tensor
If input_tensor is provided, this will return a tensor. Otherwise,
this will return a Keras model.
"""
left_outputs = []
for level in xrange(self.depth):
filter_num = int(self.max_filter / (2**(self.depth - level)) /
self.downsize_filters_factor)
if level == 0:
left_outputs += [
Conv3D(filter_num,
self.filter_shape,
activation=self.activation,
padding=self.padding)(self.inputs)
]
left_outputs[level] = Conv3D(2 * filter_num,
self.filter_shape,
activation=self.activation,
padding=self.padding)(
left_outputs[level])
else:
left_outputs += [
MaxPooling3D(pool_size=self.pool_size)(left_outputs[level -
1])
]
left_outputs[level] = Conv3D(filter_num,
self.filter_shape,
activation=self.activation,
padding=self.padding)(
left_outputs[level])
left_outputs[level] = Conv3D(2 * filter_num,
self.filter_shape,
activation=self.activation,
padding=self.padding)(
left_outputs[level])
if self.dropout is not None and self.dropout != 0:
left_outputs[level] = Dropout(self.dropout)(
left_outputs[level])
if self.batch_norm:
left_outputs[level] = BatchNormalization()(left_outputs[level])
right_outputs = [left_outputs[self.depth - 1]]
for level in xrange(self.depth):
filter_num = int(self.max_filter / (2**(level)) /
self.downsize_filters_factor)
if level > 0:
right_outputs += [
UpConvolution(pool_size=self.pool_size)(
right_outputs[level - 1])
]
right_outputs[level] = concatenate([
right_outputs[level], left_outputs[self.depth - level - 1]
],
axis=4)
right_outputs[level] = Conv3D(filter_num,
self.filter_shape,
activation=self.activation,
padding=self.padding)(
right_outputs[level])
right_outputs[level] = Conv3D(int(filter_num / 2),
self.filter_shape,
activation=self.activation,
padding=self.padding)(
right_outputs[level])
else:
continue
if self.dropout is not None and self.dropout != 0:
right_outputs[level] = Dropout(self.dropout)(
right_outputs[level])
if self.batch_norm:
right_outputs[level] = BatchNormalization()(
right_outputs[level])
output_layer = Conv3D(int(self.num_outputs),
(1, 1, 1))(right_outputs[-1])
# TODO: Brainstorm better way to specify outputs
if self.input_tensor is not None:
return output_layer
if self.output_type == 'regression':
self.model = Model(inputs=self.inputs, outputs=output_layer)
self.model.compile(optimizer=Nadam(lr=self.initial_learning_rate),
loss='mean_squared_error',
metrics=['mean_squared_error'])
if self.output_type == 'binary_label':
act = Activation('sigmoid')(output_layer)
self.model = Model(inputs=self.inputs, outputs=act)
self.model.compile(optimizer=Nadam(lr=self.initial_learning_rate),
loss=dice_coef_loss,
metrics=[dice_coef])
if self.output_type == 'categorical_label':
act = Activation('softmax')(output_layer)
self.model = Model(inputs=self.inputs, outputs=act)
self.model.compile(optimizer=Nadam(lr=self.initial_learning_rate),
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
return self.model
def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None,
input_data=None, expected_output=None,
expected_output_dtype=None, fixed_batch_size=False, tolerance=1e-3):
"""Test routine for a layer with a single input tensor
and single output tensor.
"""
# generate input data
if input_data is None:
assert input_shape
if not input_dtype:
input_dtype = K.floatx()
input_data_shape = list(input_shape)
for i, e in enumerate(input_data_shape):
if e is None:
input_data_shape[i] = np.random.randint(1, 4)
input_data = (10 * np.random.random(input_data_shape))
input_data = input_data.astype(input_dtype)
else:
if input_shape is None:
input_shape = input_data.shape
if input_dtype is None:
input_dtype = input_data.dtype
if expected_output_dtype is None:
expected_output_dtype = input_dtype
# instantiation
layer = layer_cls(**kwargs)
# test get_weights , set_weights at layer level
weights = layer.get_weights()
layer.set_weights(weights)
# test and instantiation from weights
if 'weights' in inspect.getargspec(layer_cls.__init__):
kwargs['weights'] = weights
layer = layer_cls(**kwargs)
# test in functional API
if fixed_batch_size:
x = Input(batch_shape=input_shape, dtype=input_dtype)
else:
x = Input(shape=input_shape[1:], dtype=input_dtype)
y = layer(x)
assert K.dtype(y) == expected_output_dtype
# check shape inference
model = Model(x, y)
expected_output_shape = layer.compute_output_shape(input_shape)
actual_output = model.predict(input_data)
actual_output_shape = actual_output.shape
for expected_dim, actual_dim in zip(expected_output_shape,
actual_output_shape):
if expected_dim is not None:
assert expected_dim == actual_dim
if expected_output is not None:
if tolerance is not None:
assert_allclose(actual_output, expected_output, rtol=tolerance)
# test serialization, weight setting at model level
model_config = model.get_config()
recovered_model = Model.from_config(model_config)
if model.weights:
weights = model.get_weights()
recovered_model.set_weights(weights)
_output = recovered_model.predict(input_data)
if tolerance is not None:
assert_allclose(_output, actual_output, rtol=tolerance)
# test training mode (e.g. useful for dropout tests)
model.compile('rmsprop', 'mse')
model.train_on_batch(input_data, actual_output)
# test as first layer in Sequential API
layer_config = layer.get_config()
layer_config['batch_input_shape'] = input_shape
layer = layer.__class__.from_config(layer_config)
model = Sequential()
model.add(layer)
actual_output = model.predict(input_data)
actual_output_shape = actual_output.shape
for expected_dim, actual_dim in zip(expected_output_shape,
actual_output_shape):
if expected_dim is not None:
assert expected_dim == actual_dim
if expected_output is not None:
if tolerance is not None:
assert_allclose(actual_output, expected_output, rtol=1e-3)
# test serialization, weight setting at model level
model_config = model.get_config()
recovered_model = Sequential.from_config(model_config)
if model.weights:
weights = model.get_weights()
recovered_model.set_weights(weights)
_output = recovered_model.predict(input_data)
if tolerance is not None:
assert_allclose(_output, actual_output, rtol=1e-3)
# test training mode (e.g. useful for dropout tests)
model.compile('rmsprop', 'mse')
model.train_on_batch(input_data, actual_output)
# for further checks in the caller function
return actual_output
def isensee2017_model_dil3(input_shape=(4, 128, 128, 128),
n_base_filters=16,
depth=5,
dropout_rate=0.3,
n_segmentation_levels=3,
n_labels=4,
optimizer="Adam",
initial_learning_rate=5e-4,
loss_function=weighted_dice_coefficient_loss,
activation_name="sigmoid"):
"""
This function builds a model proposed by Isensee et al. for the BRATS 2017 competition:
https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf
This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical
Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf
:param input_shape:
:param n_base_filters:
:param depth:
:param dropout_rate:
:param n_segmentation_levels:
:param n_labels:
:param optimizer:
:param initial_learning_rate:
:param loss_function:
:param activation_name:
:return:
"""
inputs = Input(input_shape)
current_layer = inputs
level_output_layers = list()
level_filters = list()
for level_number in range(depth):
n_level_filters = (2**level_number) * n_base_filters
level_filters.append(n_level_filters)
if current_layer is inputs:
in_conv = create_convolution_block(current_layer, n_level_filters)
else:
in_conv = create_convolution_block(current_layer,
n_level_filters,
strides=(2, 2, 2),
dilation_rate=(1, 1, 1))
context_output_layer = create_context_module(in_conv,
n_level_filters,
dropout_rate=dropout_rate)
summation_layer = Add()([in_conv, context_output_layer])
level_output_layers.append(summation_layer)
current_layer = summation_layer
segmentation_layers = list()
for level_number in range(depth - 2, -1, -1):
up_sampling = create_up_sampling_module(current_layer,
level_filters[level_number])
concatenation_layer = concatenate(
[level_output_layers[level_number], up_sampling], axis=1)
localization_output = create_localization_module(
concatenation_layer, level_filters[level_number])
current_layer = localization_output
if level_number < n_segmentation_levels:
segmentation_layers.insert(
0,
create_convolution_block(current_layer,
n_filters=n_labels,
kernel=(1, 1, 1)))
output_layer = None
for level_number in reversed(range(n_segmentation_levels)):
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling3D(size=(2, 2, 2))(output_layer)
activation_block = Activation(activation_name)(output_layer)
model = Model(inputs=inputs, outputs=activation_block)
if optimizer == "YellowFin":
# import and instantiate yellowFin optimizer
from .yellowfinkeras.yellowfin import YFOptimizer
from keras.optimizers import TFOptimizer
# define your optimizer
optimizer = TFOptimizer(YFOptimizer())
model.compile(optimizer=optimizer, loss=loss_function)
elif optimizer == "Adam":
from keras.optimizers import Adam
optimizer = Adam
model.compile(optimizer=optimizer(lr=initial_learning_rate),
loss=loss_function)
else:
raise ValueError(
str(error) + "\n\nYou can use only Adam or YellowFin optimizer\n")
return model
def create_model(desired_sample_rate, dilation_depth, nb_stacks):
# desired_sample_rate = 4410
nb_output_bins = 4
# nb_filters = 256
nb_filters = 64
# dilation_depth = 9 #
# nb_stacks = 1
use_bias = False
res_l2 = 0
final_l2 = 0
fragment_length = 488 + compute_receptive_field_(
desired_sample_rate, dilation_depth, nb_stacks)[0]
fragment_stride = 488
use_skip_connections = True
learn_all_outputs = True
def residual_block(x):
original_x = x
# TODO: initalization, regularization?
# Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet.
tanh_out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_tanh_s%d' %
(2**i, s),
activation='tanh',
W_regularizer=l2(res_l2))(x)
x = layers.Dropout(0.2)(x)
sigm_out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_sigm_s%d' %
(2**i, s),
activation='sigmoid',
W_regularizer=l2(res_l2))(x)
x = layers.Merge(mode='mul', name='gated_activation_%d_s%d' %
(i, s))([tanh_out, sigm_out])
res_x = layers.Convolution1D(nb_filters,
1,
border_mode='same',
bias=use_bias,
W_regularizer=l2(res_l2))(x)
skip_x = layers.Convolution1D(nb_filters,
1,
border_mode='same',
bias=use_bias,
W_regularizer=l2(res_l2))(x)
res_x = layers.Merge(mode='sum')([original_x, res_x])
return res_x, skip_x
input = Input(shape=(fragment_length, nb_output_bins), name='input_part')
out = input
skip_connections = []
out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=1,
border_mode='valid',
causal=True,
name='initial_causal_conv')(out)
for s in range(nb_stacks):
for i in range(0, dilation_depth + 1):
out, skip_out = residual_block(out)
skip_connections.append(skip_out)
if use_skip_connections:
out = layers.Merge(mode='sum')(skip_connections)
out = layers.PReLU()(out)
# out = layers.Convolution1D(nb_filter=256, filter_length=1, border_mode='same',
# W_regularizer=l2(final_l2))(out)
out = layers.Convolution1D(nb_filter=nb_output_bins,
filter_length=3,
border_mode='same')(out)
out = layers.Dropout(0.5)(out)
out = layers.PReLU()(out)
out = layers.Convolution1D(nb_filter=nb_output_bins,
filter_length=3,
border_mode='same')(out)
if not learn_all_outputs:
raise DeprecationWarning(
'Learning on just all outputs is wasteful, now learning only inside receptive field.'
)
out = layers.Lambda(
lambda x: x[:, -1, :], output_shape=(out._keras_shape[-1], ))(
out) # Based on gif in deepmind blog: take last output?
# out = layers.Activation('softmax', name="output_softmax")(out)
out = layers.PReLU()(out)
# out = layers.Activation('sigmoid', name="output_sigmoid")(out)
out = layers.Flatten()(out)
predictions = layers.Dense(919, activation='sigmoid', name='fc1')(out)
model = Model(input, predictions)
# x = model.output
# x = layers.Flatten()(x)
# # x = layers.Dense(output_dim=1024)(x)
# # x = layers.PReLU()(x)
# # x = layers.Dropout(0.5)(x)
# # x = layers.Dense(output_dim=919)(x)
# # x = layers.Activation('sigmoid')(x)
# model = Model(input=model.input, output=predictions)
receptive_field, receptive_field_ms = compute_receptive_field_(
desired_sample_rate, dilation_depth, nb_stacks)
_log.info('Receptive Field: %d (%dms)' %
(receptive_field, int(receptive_field_ms)))
return model
print("Building model...")
# Encoder
source = Input(shape=(None, ), dtype='int32', name='source')
embedded = Embedding(output_dim=128,
input_dim=train.source_vocab_size(),
mask_zero=True)(source)
last_hid = Bidirectional(LSTM(output_dim=128))(embedded)
# Decoder
repeated = RepeatVector(train.target.padded.shape[1])(last_hid)
decoder = LSTM(output_dim=128, return_sequences=True,
name="decoder1")(repeated)
decoder = LSTM(output_dim=128, return_sequences=True, name="decoder2")(decoder)
output = TimeDistributed(
Dense(output_dim=train.target_vocab_size(), activation='softmax'))(decoder)
model = Model([source], output=[output])
print("Compiling model...")
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=[all_acc])
print("Training...")
train_sources, train_targets = train.all()
dev_sources, dev_targets = dev.all()
model.fit(train_sources,
train_targets[:, :, None],
nb_epoch=50,
validation_data=(dev_sources, dev_targets[:, :, None]),
callbacks=[
ModelCheckpoint("model.{epoch:02d}-{val_loss:.2f}.hdf5",
def define(self,
features_model: str,
n_output: int = 2,
dropout: float = 1.,
n_units: int = 256,
lstm_layers: int = 1,
rec_type: str = "gru",
base_model=None):
"""
Define two_stream_rnn model architecture for trained two_stream network + rnn module from scratch
:param features_model: already trained static model
:param n_output: number of network outputs
:param dropout: dropout value
:param n_units: number of RNN units
:param lstm_layers: number of RNN layers
:param rec_type: type of recurrent layer (GRU or LSTM)
:param base_model: Base model whose architecture and weights are used for individual weights. In this case
base model is just used to know the data input sizes, as base_model is already used within
"features_model"
"""
# Initialize weights
weight_init = glorot_uniform(seed=3)
# Load pre-trained model
print("Loading model... ", features_model)
cnn_model = features_model #load_model(features_model, custom_objects=self.custom_objects)
face_input = base_model.input_size[input_type.FACE]
eyes_input = base_model.input_size[input_type.EYES]
landmarks_input = base_model.input_size[input_type.LANDMARKS]
face_input_seq = Input(shape=(None, face_input[0], face_input[1],
face_input[2]),
name='seq_input_' + input_type.FACE.value)
eyes_input_seq = Input(shape=(None, eyes_input[0], eyes_input[1],
eyes_input[2]),
name='seq_input_' + input_type.EYES.value)
landmarks_input_seq = Input(shape=(None, landmarks_input[0]),
name='seq_' + input_type.LANDMARKS.value)
model_input = [face_input_seq, eyes_input_seq, landmarks_input_seq]
# --- INDIVIDUAL and FUSION ---
face_counter = 0
eyes_counter = 0
for layer in cnn_model.layers:
layer.trainable = False
if input_type.FACE.value in layer.name and "dp" not in layer.name:
if face_counter == 0:
fx = face_input_seq
face_counter += 1
else:
# The layer is not trained when using lambda.
# fx = TimeDistributed(layer)(fx)
fx = TimeDistributed(Lambda(lambda x: layer(x)))(fx)
elif input_type.EYES.value in layer.name and "dp" not in layer.name:
if eyes_counter == 0:
ex = eyes_input_seq
eyes_counter += 1
else:
# ex = TimeDistributed(layer)(ex)
ex = TimeDistributed(Lambda(lambda x: layer(x)))(ex)
elif "concatenate" in layer.name:
x = concatenate([fx, ex, landmarks_input_seq])
else:
if "out" not in layer.name and "input" not in layer.name:
layer.trainable = True
x = TimeDistributed(layer)(x)
# x = TimeDistributed(Lambda(lambda x: layer(x)))(x)
# --- TEMPORAL ---
for i in range(lstm_layers):
namei = "rec_l" + str(i)
named = "rec_dp" + str(i)
if lstm_layers > 1 and i == 0:
return_sequences = True
else:
return_sequences = False
# if dropout < 1.:
# x = Dropout(dropout, seed=0, name=named)(x)
num_units = int(n_units / (int(i) + 1))
print("Num units lstm/gru: ", num_units)
if rec_type == "lstm":
if len(K.tensorflow_backend._get_available_gpus()) > 0:
x = CuDNNLSTM(num_units,
name=namei,
return_sequences=return_sequences)(x)
else:
x = LSTM(num_units,
name=namei,
return_sequences=return_sequences)(x)
else:
if len(K.tensorflow_backend._get_available_gpus()) > 0:
x = CuDNNGRU(num_units,
name=namei,
return_sequences=return_sequences)(x)
else:
x = GRU(num_units,
name=namei,
return_sequences=return_sequences)(x)
if i < (lstm_layers - 1) and dropout < 1.:
x = Dropout(dropout, seed=0, name=named)(x)
# --- LINEAR REGRESSION ---
out = Dense(n_output, kernel_initializer=weight_init, name='out')(x)
self.model = Model(inputs=model_input, outputs=out)
self.model.summary()
print(len(self.model.layers))
print([n.name for n in self.model.layers])
del cnn_model
gc.collect()
def att_original_unet_model(input_shape,
pool_size=(2, 2, 2),
n_labels=1,
initial_learning_rate=0.00001,
deconvolution=False,
depth=4,
n_base_filters=32,
metrics=dice_coefficient,
batch_normalization=False,
activation_name="sigmoid"):
"""
Builds the 3D UNet Keras model.f
:param metrics: List metrics to be calculated during model training (default is dice coefficient).
:param n_base_filters: The number of filters that the first layer in the convolution network will have. Following
layers will contain a multiple of this number. Lowering this number will likely reduce the amount of memory required
to train the model.
:param depth: indicates the depth of the U-shape for the model. The greater the depth, the more max pooling
layers will be added to the model. Lowering the depth may reduce the amount of memory required for training.
:param input_shape: Shape of the input data (n_chanels, x_size, y_size, z_size). The x, y, and z sizes must be
divisible by the pool size to the power of the depth of the UNet, that is pool_size^depth.
:param pool_size: Pool size for the max pooling operations.
:param n_labels: Number of binary labels that the model is learning.
:param initial_learning_rate: Initial learning rate for the model. This will be decayed during training.
:param deconvolution: If set to True, will use transpose convolution(deconvolution) instead of up-sampling. This
increases the amount memory required during training.
:return: Untrained 3D UNet Model
"""
inputs = Input(input_shape)
current_layer = inputs
levels = list()
level_filters = list()
for level_number in range(depth): # depth=4; [0,1,2,3]
n_level_filters = list()
n_level_filters.append(
(2**level_number) * n_base_filters) # 2^level_num * 16
n_level_filters.append(n_level_filters[0] * 2)
level_filters.append(n_level_filters)
# add levels with max pooling
layer1 = create_convolution_block(
input_layer=current_layer,
n_filters=n_level_filters[0],
batch_normalization=batch_normalization)
layer2 = create_convolution_block(
input_layer=layer1,
n_filters=n_level_filters[1],
batch_normalization=batch_normalization)
if level_number < depth - 1:
current_layer = MaxPooling3D(pool_size=pool_size)(layer2)
levels.append([layer1, layer2, current_layer])
else: # 最下一层
current_layer = layer2
levels.append([layer1, layer2])
for level_number in range(depth - 2, -1, -1): # [2,1,0]
gating = gating_signal(levels[level_number + 1][1],
level_filters[level_number][1], False)
att = attention_block(levels[level_number][1], gating,
level_filters[level_number][1])
up_convolution = get_up_convolution(
pool_size=pool_size,
deconvolution=deconvolution,
n_filters=level_filters[level_number + 1][1])(current_layer)
concat = concatenate([up_convolution, att], axis=1)
current_layer = create_convolution_block(
n_filters=level_filters[level_number][1],
input_layer=concat,
batch_normalization=batch_normalization)
current_layer = create_convolution_block(
n_filters=level_filters[level_number][1],
input_layer=current_layer,
batch_normalization=batch_normalization)
# sigmoid激活输出
final_convolution = Conv3D(n_labels, (1, 1, 1))(current_layer)
act = Activation(activation_name)(final_convolution)
model = Model(inputs=inputs, outputs=act)
model.compile(optimizer=Adam(lr=initial_learning_rate),
loss=dice_coefficient_loss)
# 返回模型
return model
if (i == 0):
x = identity_block(x, 3, 32, stage=3, block=str(i), subsumpling=True)
else:
x = identity_block(x, 3, 32, stage=3, block=str(i))
for i in range(n):
if (i == 0):
x = identity_block(x, 3, 64, stage=4, block=str(i), subsumpling=True)
else:
x = identity_block(x, 3, 64, stage=4, block=str(i))
x = AveragePooling2D((2, 2), name='avg_pool')(x)
x = Flatten()(x)
x = Dense(nb_classes, activation='softmax', name='fc1000')(x)
model = Model(img_input, x)
model.summary()
datagen = ImageDataGenerator(
featurewise_center=True, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
def build_model(fragment_length, nb_filters, nb_output_bins, dilation_depth,
nb_stacks, use_skip_connections, learn_all_outputs, _log,
desired_sample_rate, use_bias, res_l2, final_l2):
def residual_block(x):
original_x = x
# TODO: initalization, regularization?
# Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet.
tanh_out = layers.AtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_tanh_s%d' %
(2**i, s),
activation='tanh',
W_regularizer=l2(res_l2))(x)
sigm_out = layers.AtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_sigm_s%d' %
(2**i, s),
activation='sigmoid',
W_regularizer=l2(res_l2))(x)
x = layers.Merge(mode='mul', name='gated_activation_%d_s%d' %
(i, s))([tanh_out, sigm_out])
res_x = layers.Convolution1D(nb_filters,
1,
border_mode='same',
bias=use_bias,
W_regularizer=l2(res_l2))(x)
skip_x = layers.Convolution1D(nb_filters,
1,
border_mode='same',
bias=use_bias,
W_regularizer=l2(res_l2))(x)
res_x = layers.Merge(mode='sum')([original_x, res_x])
return res_x, skip_x
input = Input(shape=(fragment_length, nb_output_bins), name='input_part')
out = input
skip_connections = []
out = layers.AtrousConvolution1D(nb_filters,
2,
atrous_rate=1,
border_mode='valid',
causal=True,
name='initial_causal_conv')(out)
for s in range(nb_stacks):
for i in range(0, dilation_depth + 1):
out, skip_out = residual_block(out)
skip_connections.append(skip_out)
if use_skip_connections:
out = layers.Merge(mode='sum')(skip_connections)
out = layers.Activation('relu')(out)
out = layers.Convolution1D(nb_output_bins,
1,
border_mode='same',
W_regularizer=l2(final_l2))(out)
out = layers.Activation('relu')(out)
out = layers.Convolution1D(nb_output_bins, 1, border_mode='same')(out)
if not learn_all_outputs:
raise DeprecationWarning(
'Learning on just all outputs is wasteful, now learning only inside receptive field.'
)
out = layers.Lambda(
lambda x: x[:, -1, :], output_shape=(out._keras_shape[-1], ))(
out) # Based on gif in deepmind blog: take last output?
out = layers.Activation('softmax', name="output_softmax")(out)
model = Model(input, out)
receptive_field, receptive_field_ms = compute_receptive_field()
_log.info('Receptive Field: %d (%dms)' %
(receptive_field, int(receptive_field_ms)))
return model
Y_test = np_utils.to_categorical(y_test, nb_classes)
nb_train_samples = X_train.shape[0]
nb_validation_samples = X_test.shape[0]
print nb_train_samples
# Extract the last layer from third block of vgg16 model
last = base_model.get_layer('block3_pool').output
# Add classification layers on top of it
x = Flatten()(last)
# x = Dense(256, activation='relu')(x)
# x = Dropout(0.5)(x)
pred = Dense(10, activation='softmax')(x)
model = Model(base_model.input, pred)
# set the base model's layers to non-trainable
# uncomment next two lines if you don't want to
# train the base model
for layer in base_model.layers:
layer.trainable = False
# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
# model.compile(loss='binary_crossentropy',
# optimizer=optimizers.SGD(lr=1e-3, momentum=0.9),
# metrics=['accuracy'])
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
data = np.load('face_data.npz')
x, y = data['x'], data['y']
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=0.20,
random_state=0)
a_x = x_train[y_train == 0]
a_y = y_train[y_train == 0]
layer_name = 'flatten_1' # edit this line
resnet = VGGFace(model='resnet50', input_shape=(224, 224, 3))
out = resnet.get_layer(layer_name).output
resnet_features = Model(resnet.input, out)
a_x_feats = resnet_features.predict(a_x)
#print (a_x_feats.shape)
dists = []
for i in range(400):
ind1 = np.random.randint(0, a_x_feats.shape[0])
ind2 = np.random.randint(0, a_x_feats.shape[0])
dist = np.linalg.norm(a_x_feats[ind1, :] - a_x_feats[ind2, :])
#print (dist)
dists.append(dist)
dists = np.array(dists)
def base_encoder(input_shape, hidden_dim, l2_reg):
input = Input(shape=input_shape)
x = Dense(hidden_dim, activation='relu', input_shape=input_shape, kernel_regularizer=l2(l2_reg))(input)
x = Dense(hidden_dim//2, activation='relu', kernel_regularizer=l2(l2_reg))(x)
return Model(input, x)
img = image.img_to_array(
image.load_img(img_file, target_size=(img_w, img_h)))
Img_data[k, ...] = img
Img_Label[k, ...] = Val_label[(start_idx + k), ...]
Img_data = preprocess_input(Img_data)
yield Img_data, Img_Label
image_input = Input(shape=(img_w, img_h, channels))
vggface_model = VGGFace(input_tensor=image_input, include_top=False)
last_layer = vggface_model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(4096, activation='relu', name='fc6')(x)
x = Dense(4096, activation='relu', name='fc7')(x)
out = Dense(classes, activation='softmax', name='fc8')(x)
keras_vggface_IJBA = Model(image_input, out)
sgd = SGD(lr=0.001, decay=0.0, momentum=0.9, nesterov=False)
print(keras_vggface_IJBA.summary())
keras_vggface_IJBA.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
keras_vggface_IJBA_json = keras_vggface_IJBA.to_json()
with open(
"/home/zhaojian/Keras/Projects/GAN/models/VGGFACE/keras_vggface_IJBA.json",
"w") as json_file:
json_file.write(keras_vggface_IJBA_json)
# checkpoint
filepath = "/home/zhaojian/Keras/Projects/GAN/models/VGGFACE/weights-improvement-{epoch:02d}-{val_acc:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
from keras.layers import Flatten, Dense, Input
from keras_vggface.vggface import VGGFace
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
#custom parameters
nb_class = 1
hidden_dim = 512
vgg_model = VGGFace(include_top=False, input_shape=(224, 224, 3))
last_layer = vgg_model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(hidden_dim, activation='relu', name='fc6')(x)
x = Dense(hidden_dim, activation='relu', name='fc7')(x)
out = Dense(nb_class, activation='softmax', name='fc8')(x)
custom_vgg_model = Model(vgg_model.input, out)
for layer in custom_vgg_model.layers[:19]:
layer.trainable = False
custom_vgg_model.compile(loss='binary_crossentropy',
optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
train_data_dir = 'cropped_data/train'
validation_data_dir = 'cropped_data/test'
nb_train_samples = 419
nb_validation_samples = 100
epochs = 5
batch_size = 16
def get_unet_3d(vol_x,
vol_y,
vol_z,
chn,
optimizer=Adam(lr=0.00001),
loss=dice_coef_loss,
metrics=[dice_coef]):
print('Data format: ' + K.image_data_format())
if K.image_data_format() == 'channels_first':
input_dims = (chn, vol_x, vol_y, vol_z)
feat_axis = 1
else:
input_dims = (vol_x, vol_y, vol_z, chn)
feat_axis = 4
# u-net model
inputs = Input(shape=input_dims)
conv1 = Conv3D(32, (3, 3, 3), activation=None, padding='same')(
inputs) #Conv3D --> (filters, kernel_size, ...)
conv1 = BatchNormalization(axis=feat_axis, scale=False)(conv1)
conv1 = Activation('relu')(conv1)
conv1 = Conv3D(64, (3, 3, 3), activation=None, padding='same')(conv1)
conv1 = BatchNormalization(axis=feat_axis, scale=False)(conv1)
conv1 = Activation('relu')(conv1)
pool1 = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2))(conv1)
conv2 = Conv3D(64, (3, 3, 3), activation=None, padding='same')(pool1)
conv2 = BatchNormalization(axis=feat_axis, scale=False)(conv2)
conv2 = Activation('relu')(conv2)
conv2 = Conv3D(128, (3, 3, 3), activation=None, padding='same')(conv2)
conv2 = BatchNormalization(axis=feat_axis, scale=False)(conv2)
conv2 = Activation('relu')(conv2)
pool2 = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2))(conv2)
conv3 = Conv3D(128, (3, 3, 3), activation=None, padding='same')(pool2)
conv3 = BatchNormalization(axis=feat_axis, scale=False)(conv3)
conv3 = Activation('relu')(conv3)
conv3 = Conv3D(256, (3, 3, 3), activation=None, padding='same')(conv3)
conv3 = BatchNormalization(axis=feat_axis, scale=False)(conv3)
conv3 = Activation('relu')(conv3)
pool3 = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2))(conv3)
conv4 = Conv3D(256, (3, 3, 3), activation=None, padding='same')(pool3)
conv4 = BatchNormalization(axis=feat_axis, scale=False)(conv4)
conv4 = Activation('relu')(conv4)
conv4 = Conv3D(512, (3, 3, 3), activation=None, padding='same')(conv4)
conv4 = BatchNormalization(axis=feat_axis, scale=False)(conv4)
conv4 = Activation('relu')(conv4)
up1 = UpSampling3D(size=(2, 2, 2))(conv4)
up1 = Concatenate(axis=feat_axis)([conv3, up1])
upconv1 = Conv3D(256, (3, 3, 3), activation=None, padding='same')(up1)
upconv1 = BatchNormalization(axis=feat_axis, scale=False)(upconv1)
upconv1 = Activation('relu')(upconv1)
upconv1 = Conv3D(256, (3, 3, 3), activation=None, padding='same')(upconv1)
upconv1 = BatchNormalization(axis=feat_axis, scale=False)(upconv1)
upconv1 = Activation('relu')(upconv1)
up2 = UpSampling3D(size=(2, 2, 2))(upconv1)
up2 = Concatenate(axis=feat_axis)([conv2, up2])
upconv2 = Conv3D(128, (3, 3, 3), activation=None, padding='same')(up2)
upconv2 = BatchNormalization(axis=feat_axis, scale=False)(upconv2)
upconv2 = Activation('relu')(upconv2)
upconv2 = Conv3D(128, (3, 3, 3), activation=None, padding='same')(upconv2)
upconv2 = BatchNormalization(axis=feat_axis, scale=False)(upconv2)
upconv2 = Activation('relu')(upconv2)
up3 = UpSampling3D(size=(2, 2, 2))(upconv2)
up3 = Concatenate(axis=feat_axis)([conv1, up3])
upconv3 = Conv3D(64, (3, 3, 3), activation=None, padding='same')(up3)
upconv3 = BatchNormalization(axis=feat_axis, scale=False)(upconv3)
upconv3 = Activation('relu')(upconv3)
upconv3 = Conv3D(64, (3, 3, 3), activation=None, padding='same')(upconv3)
upconv3 = BatchNormalization(axis=feat_axis, scale=False)(upconv3)
upconv3 = Activation('relu')(upconv3)
conv_final = Conv3D(4, (3, 3, 3), activation='sigmoid',
padding='same')(upconv3)
model = Model(inputs=inputs, outputs=conv_final)
model.summary()
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
return model
# Bring coordinates and distance in roughly the same scale
base_precision = 1e4
base = np.floor(base_precision * training_vectors[:, 0:1, :])
base = np.repeat(base, 2, axis=1)
training_vectors = (base_precision * training_vectors) - base
(data_points, max_points, GEO_VECTOR_LEN) = training_vectors.shape
target_vectors = loaded['centroid_distance'][:, 0, :]
inputs = Input(name='Input', shape=(max_points, GEO_VECTOR_LEN))
# This can be a simple Dense layer of size 16 as well
#model = Flatten()(inputs)
model = LSTM(LATENT_SIZE, activation='relu')(inputs)
model = Dense(2)(model)
model = Model(inputs, model)
model.compile(loss=univariate_gaussian_loss, optimizer=OPTIMIZER)
model.summary()
callbacks = [
TensorBoard(log_dir='./tensorboard_log/' + TIMESTAMP + ' ' + SCRIPT_NAME,
write_graph=False),
DecypherAll(lambda x: str(x)),
EarlyStopping(patience=40, min_delta=1e-3)
]
history = model.fit(x=training_vectors,
y=target_vectors,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_split=TRAIN_VALIDATE_SPLIT,
retrained_model_path = 'vggface-retrained.h5'
maxAttempt = 4095
confidenceLevel = 0.99
nb_class = 5
alpha = 0.1
epoch = 50
initImage = 2 #0:random; 1:a face image; other values:blank
VGGFace_original_model = VGGFace(model='vgg16', pooling='max')
vgg_retrained = load_model(retrained_model_path)
XX = VGGFace_original_model.input
YY = VGGFace_original_model.layers[retrainLayerIndex].output
feature_extractor = Model(XX, YY)
def generate_image(input, model, intermediate_target):
loss = K.sum(K.square(model.output[0] - intermediate_target[0]))
gradients = K.gradients(loss, model.input)[0]
fn = K.function([model.input], [gradients])
for i in range(1, epoch):
grads = fn([input])
input = input - alpha * grads[0]
return input
def predictor():
model = Sequential()
def isensee2017_model(input_shape=(4, 128, 128, 128), n_base_filters=16, depth=5, dropout_rate=0.3,
n_segmentation_levels=3, n_labels=1, optimizer=Adam, initial_learning_rate=5e-4,
loss_function=dice_coefficient_loss, activation_name="sigmoid", **kargs):
"""
This function builds a model proposed by Isensee et al. for the BRATS 2017 competition:
https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf
This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical
Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf
:param input_shape:
:param n_base_filters:
:param depth:
:param dropout_rate:
:param n_segmentation_levels:
:param n_labels:
:param optimizer:
:param initial_learning_rate:
:param loss_function:
:param activation_name:
:return:
"""
metrics = ['binary_accuracy', vod_coefficient]
if loss_function != dice_coefficient_loss:
metrics += [dice_coefficient]
inputs = Input(input_shape)
inputs_p = Permute((3, 1, 2))(inputs)
current_layer = inputs_p
level_output_layers = list()
level_filters = list()
for level_number in range(depth):
n_level_filters = (2 ** level_number) * n_base_filters
level_filters.append(n_level_filters)
if current_layer is inputs_p:
in_conv = create_convolution_block(current_layer, n_level_filters)
else:
in_conv = create_convolution_block(current_layer, n_level_filters, strides=(2, 2))
context_output_layer = create_context_module(in_conv, n_level_filters, dropout_rate=dropout_rate)
summation_layer = Add()([in_conv, context_output_layer])
level_output_layers.append(summation_layer)
current_layer = summation_layer
segmentation_layers = list()
for level_number in range(depth - 2, -1, -1):
up_sampling = create_up_sampling_module(current_layer, level_filters[level_number])
concatenation_layer = concatenate([level_output_layers[level_number], up_sampling], axis=1)
localization_output = create_localization_module(concatenation_layer, level_filters[level_number])
current_layer = localization_output
if level_number < n_segmentation_levels:
segmentation_layers.insert(0, Conv2D(n_labels, (1, 1))(current_layer))
output_layer = None
for level_number in reversed(range(n_segmentation_levels)):
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling2D(size=(2, 2))(output_layer)
activation_block = Activation(activation_name)(output_layer)
activation_block = Permute((2, 3, 1))(activation_block)
model = Model(inputs=inputs, outputs=activation_block)
model.compile(optimizer=optimizer(lr=initial_learning_rate), loss=loss_function,
metrics=metrics)
return model
logits = K.squeeze(logits, axis=2)
pre_softmax = K.switch(mask[0], logits, -numpy.inf)
weights = K.expand_dims(K.softmax(pre_softmax))
return K.sum(vectors * weights, axis=1)
def attn_merge_shape(input_shapes):
return (input_shapes[0][0], input_shapes[0][2])
attn = Lambda(attn_merge, output_shape=attn_merge_shape)
attn.supports_masking = True
attn.compute_mask = lambda inputs, mask: None
content_flat = attn([seq_output, attention_2])
model = Model(inputs=morph_seg, outputs=content_flat)
model.summary()
model.load_weights("weights.h5")
m_w = model.get_layer("embeddding").get_weights()
print(m_w)
m_vectors = {}
for i in range(len(m_w[0])):
if not indices_morph[i] == '###':
m_vectors[indices_morph[i]] = m_w[0][i]
import cPickle as pickle
with open('vectors.p', 'wb') as fp:
def unet_model_2d(input_shape, n_labels, batch_normalization=False, initial_learning_rate=0.00001,
metrics=m.dice_coefficient):
"""
input_shape:without batch_size,(img_height,img_width,img_depth)
metrics:
"""
inputs = Input(input_shape)
down_layer = []
layer = inputs
# down_layer_1
layer = res_block_v2(layer, 64, batch_normalization=batch_normalization)
down_layer.append(layer)
layer = MaxPooling2D(pool_size=[2, 2], strides=[2, 2])(layer)
print(str(layer.get_shape()))
# down_layer_2
layer = res_block_v2(layer, 128, batch_normalization=batch_normalization)
down_layer.append(layer)
layer = MaxPooling2D(pool_size=[2, 2], strides=[2, 2])(layer)
print(str(layer.get_shape()))
# down_layer_3
layer = res_block_v2(layer, 256, batch_normalization=batch_normalization)
down_layer.append(layer)
layer = MaxPooling2D(pool_size=[2, 2], strides=[2, 2])(layer)
print(str(layer.get_shape()))
# down_layer_4
layer = res_block_v2(layer, 512, batch_normalization=batch_normalization)
down_layer.append(layer)
layer = MaxPooling2D(pool_size=[2, 2], strides=[2, 2])(layer)
print(str(layer.get_shape()))
# bottle_layer
layer = res_block_v2(layer, 1024, batch_normalization=batch_normalization)
print(str(layer.get_shape()))
# up_layer_4
layer = up_and_concate(layer, down_layer[3])
layer = res_block_v2(layer, 512, batch_normalization=batch_normalization)
print(str(layer.get_shape()))
# up_layer_3
layer = up_and_concate(layer, down_layer[2])
layer = res_block_v2(layer, 256, batch_normalization=batch_normalization)
print(str(layer.get_shape()))
# up_layer_2
layer = up_and_concate(layer, down_layer[1])
layer = res_block_v2(layer, 128, batch_normalization=batch_normalization)
print(str(layer.get_shape()))
# up_layer_1
layer = up_and_concate(layer, down_layer[0])
layer = res_block_v2(layer, 64, batch_normalization=batch_normalization)
print(str(layer.get_shape()))
# score_layer
layer = Conv2D(n_labels, [1, 1], strides=[1, 1])(layer)
print(str(layer.get_shape()))
# softmax
layer = Activation('softmax')(layer)
print(str(layer.get_shape()))
outputs = layer
model = Model(inputs=inputs, outputs=outputs)
metrics = [metrics]
model = multi_gpu_model(model, gpus=4)
model.compile(optimizer=Adam(lr=initial_learning_rate), loss=m.dice_coefficient_loss, metrics=metrics)
return model
def char_bielugru(word_input_size, word_embedding_size, char_input_size,
char_embedding_size, sequence_embedding_size, n_tags,
word_dropout, rnn_dropout_W, rnn_dropout_U, dropout, l2,
word_embedding_weights, **kwargs):
# define network inputs: words and character indices
text_input = Input(shape=(None, ), dtype='int32', name='text_input')
char_input = Input(shape=(
None,
None,
), dtype='int32', name='char_input')
# map word indices to vector representations
word_embeddings = Embedding(input_dim=word_input_size,
output_dim=word_embedding_size,
weights=word_embedding_weights,
name="word_embeddings")(text_input)
word_embeddings = Dropout(word_dropout)(word_embeddings)
# map each character for each word to its vector representation
char_embedding_layer = Embedding(input_dim=char_input_size,
output_dim=char_embedding_size,
name="char_embeddings")
char_embeddings = BetterTimeDistributed(char_embedding_layer)(char_input)
char_embeddings = Dropout(word_dropout)(char_embeddings)
##################
# apply char-level BiGRU to every word
char_word_model = Bidirectional(ELUGRU(char_embedding_size,
return_sequences=False),
merge_mode="concat")
char_word_embeddings = BetterTimeDistributed(char_word_model)(
char_embeddings)
char_word_embeddings = Dropout(dropout)(char_word_embeddings)
# project final states to fixed size representation
char_word_embeddings = BetterTimeDistributed(
Dense(char_embedding_size,
kernel_regularizer=L1L2(l2=l2)))(char_word_embeddings)
##################
# combine word and character emebeddings
sequence_embedding = concatenate([word_embeddings, char_word_embeddings])
# apply text level BIGRU
bidirectional_tag_sequence_output = Bidirectional(
ELUGRU(sequence_embedding_size / 2,
return_sequences=True,
dropout=rnn_dropout_W,
recurrent_dropout=rnn_dropout_U),
merge_mode="concat")(sequence_embedding)
# project hidden states to IOB tags
tag_sequence_output = TimeDistributed(
Dense(n_tags, activation='softmax', kernel_regularizer=L1L2(l2=l2)),
name="aspect_output")(bidirectional_tag_sequence_output)
# construct Model object and compile
model = Model(inputs=[text_input, char_input],
outputs=[tag_sequence_output])
adam = Adam()
model.compile(optimizer=adam,
loss={'aspect_output': "categorical_crossentropy"},
sample_weight_mode="temporal")
model._make_train_function()
model._make_predict_function()
# construct Model object to obtain the character-level verctor representation for a single word
char_word_input = Input(shape=(None, ),
dtype='int32',
name='char_word_input')
char_word_embedding = char_embedding_layer(char_word_input)
char_word_embedding = char_word_model(char_word_embedding)
char_word_model = Model(input=[char_word_input],
output=[char_word_embedding])
char_word_model._make_predict_function()
return model, char_word_model
def build_model(self, compute_output=True):
# TODO: Move this entire section to cost_functions.py
if compute_output:
if self.input_tensor is None:
if self.cost_function == 'mse':
self.model = Model(inputs=self.inputs,
outputs=self.output_layer)
self.model.compile(
optimizer=self.keras_optimizer_dict[self.optimizer](
lr=self.initial_learning_rate),
loss='mean_squared_error',
metrics=['mean_squared_error'])
elif self.cost_function == 'dice':
if self.output_activation:
self.model = Model(inputs=self.inputs,
outputs=Activation('sigmoid')(
self.output_layer))
else:
self.model = Model(inputs=self.inputs,
outputs=self.output_layer)
self.model.compile(
optimizer=self.keras_optimizer_dict[self.optimizer](
lr=self.initial_learning_rate),
loss=dice_coef_loss,
metrics=[dice_coef])
# Not Implemented
elif self.cost_function == 'multi_dice':
raise NotImplementedError
if self.output_activation:
self.model = Model(inputs=self.inputs,
outputs=Activation('sigmoid')(
self.output_layer))
else:
self.model = Model(inputs=self.inputs,
outputs=self.output_layer)
self.model.compile(
optimizer=self.keras_optimizer_dict[self.optimizer](
lr=self.initial_learning_rate),
loss=dice_coef_loss,
metrics=[dice_coef])
elif self.cost_function == 'binary_crossentropy':
if self.output_activation:
self.model = Model(inputs=self.inputs,
outputs=Activation('sigmoid')(
self.output_layer))
else:
self.model = Model(inputs=self.inputs,
outputs=self.output_layer)
self.model.compile(
optimizer=self.keras_optimizer_dict[self.optimizer](
lr=self.initial_learning_rate),
loss='binary_crossentropy',
metrics=['binary_accuracy'])
elif self.cost_function == 'categorical_crossentropy':
if self.output_activation:
self.model = Model(inputs=self.inputs,
outputs=Activation('softmax')(
self.output_layer))
else:
self.model = Model(inputs=self.inputs,
outputs=self.output_layer)
self.model.compile(
optimizer=self.keras_optimizer_dict[self.optimizer](
lr=self.initial_learning_rate),
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
elif self.cost_function == 'weighted_categorical_label':
activation = Activation('sigmoid')(self.output_layer)
activation_categorical = Lambda(
lambda arg: K.ones_like(arg) - arg)(activation)
predictions = concatenate(
[activation, activation_categorical], axis=-1)
if self.output_activation:
self.model = Model(inputs=self.inputs,
outputs=predictions)
else:
self.model = Model(inputs=self.inputs,
outputs=self.output_layer)
lossFunc = WeightedCategoricalCrossEntropy(
self.categorical_weighting)
self.model.compile(
self.keras_optimizer_dict[self.optimizer](
lr=self.initial_learning_rate),
loss=lossFunc.loss_wcc_dist,
metrics=[
lossFunc.metric_dice_dist, lossFunc.metric_acc
])
else:
print('Cost function', self.cost_function,
'not implemented.')
raise NotImplementedError
self.model_input_shape = self.model.layers[0].input_shape
self.model_output_shape = self.model.layers[-1].output_shape
return self.model
else:
self.model_input_shape = self.model.layers[0].input_shape
self.model_output_shape = self.model.layers[-1].output_shape
return self.output_layer
else:
self.model_input_shape = self.model.layers[0].input_shape
self.model_output_shape = self.model.layers[-1].output_shape
return
def build_generator(config: BEGANConfig):
decoder = build_decoder(config, name="generator_decoder")
generator = Model(decoder.inputs, decoder.outputs, name="generator")
return generator
def train_pair_predict(pair_model_path, target_train_path, pid_path,
score_path):
model = load_model(pair_model_path)
model = Model(inputs=[model.get_layer('resnet50').get_input_at(0)],
outputs=[model.get_layer('resnet50').get_output_at(0)])
train_predict(model, target_train_path, pid_path, score_path)
def build_generator(img_shape, gf):
"""U-Net Generator"""
def conv3d(layer_input, filters, f_size=(4, 4, 4), bn=True):
d = create_convolution_block(input_layer=layer_input,
n_filters=filters,
batch_normalization=bn,
strides=(2, 2, 2),
kernel_size=f_size)
return d
def deconv3d(layer_input,
skip_input,
filters,
f_size=(4, 4, 4),
drop=True):
"""Layers used during upsampling"""
u = create_convolution_block_up(input_layer=layer_input,
skip_conn=skip_input,
n_filters=filters,
batch_normalization=True,
strides=(2, 2, 2),
kernel_size=f_size,
dropout=drop)
return u
# Image input
d0 = Input(batch_shape=img_shape)
# Downsampling
e1 = conv3d(d0, gf, bn=False) # 64
e2 = conv3d(e1, gf * 2) # 128
e3 = conv3d(e2, gf * 4) # 256
e4 = conv3d(e3, gf * 8) # 512
e5 = conv3d(e4, gf * 8) # 512
# bottleneck
e6 = bottleneck(e5,
gf * 8,
batch_normalization=False,
kernel_size=(4, 4, 4),
activation='relu',
padding='same',
strides=(2, 2, 2),
instance_normalization=False) # 512
# Upsampling
u1 = deconv3d(e6, e5, gf * 8, drop=True)
u2 = deconv3d(u1, e4, gf * 8, drop=True)
u3 = deconv3d(u2, e3, gf * 4, drop=True)
u4 = deconv3d(u3, e2, gf * 2, drop=False)
u5 = deconv3d(u4, e1, gf, drop=False)
#
#
init = RandomNormal(mean=0.0, stddev=0.02) # new
u6 = Conv3DTranspose(filters=gf,
kernel_size=(4, 4, 4),
padding='same',
kernel_initializer=init,
strides=(2, 2, 2))(u5)
#
final_convolution = Conv3D(1, (1, 1, 1))(u6)
act = Activation('relu')(final_convolution)
return Model(d0, act)
def get_model_3d(kwargs):
base_filters = kwargs['base_filters']
gpus = kwargs['numgpu']
loss = kwargs['loss']
numchannel = int(len(kwargs['modalities']))
inputs = Input((None, None, None, int(numchannel)))
if kwargs['model'] == 'inception':
conv1 = Conv3D(base_filters * 8, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(inputs)
conv2 = Conv3D(base_filters * 8, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(conv1)
inception1 = Inception3d(conv2, base_filters)
inception2 = Inception3d(inception1, base_filters)
inception3 = Inception3d(inception2, base_filters)
convconcat1 = Conv3D(base_filters * 4, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(inception3)
final = Conv3D(base_filters * 4, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(convconcat1)
elif kwargs['model'] == 'unet':
final = Unet3D(inputs, base_filters)
elif kwargs['model'] == 'vnet':
final = Vnet3D(inputs, base_filters)
elif kwargs['model'] == 'fpn' or kwargs['model'] == 'panopticfpn':
reg = 0.0001
f1, f2, f3, f4, _ = FPN3D(inputs, base_filters, reg)
elif kwargs['model'] == 'densenet':
final = DenseNet3D(inputs,base_filters)
else:
sys.exit('Model must be inception/unet/vnet/fpn.')
if kwargs['model'] != 'fpn' and kwargs['model'] != 'panopticfpn':
if loss == 'bce' or loss == 'dice' or loss == 'focal':
final = Conv3D(1, (3, 3, 3), activation='sigmoid', padding='same', strides=(1, 1, 1))(final)
else:
final = Conv3D(1, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(final)
model = Model(inputs=inputs, outputs=final,name='some_unique_name')
else:
if kwargs['model'] == 'panopticfpn':
if loss == 'bce' or loss == 'dice' or loss == 'focal':
# Generate the semantic segmentation branch of panoptic FPN on top of feature extraction backbone
# Upsampling stages for F4
# U1
f4 = BatchNormalization(axis=-1)(f4)
f4 = Activation('relu')(f4)
f4 = UpSampling3D(size=(2, 2, 2), name='F4_U1')(f4)
# U2
f4 = Conv3D(base_filters*4, (3, 3, 3), padding='same', strides=(1, 1, 1), kernel_regularizer=l2(reg))(f4)
f4 = BatchNormalization(axis=-1)(f4)
f4 = Activation('relu')(f4)
f4 = UpSampling3D(size=(2, 2, 2), name='F4_U2')(f4)
# U3
f4 = Conv3D(base_filters*4, (3, 3, 3), padding='same', strides=(1, 1, 1), kernel_regularizer=l2(reg))(f4)
f4 = BatchNormalization(axis=-1)(f4)
f4 = Activation('relu')(f4)
f4 = UpSampling3D(size=(2, 2, 2), name='F4_U3')(f4)
# Prepare
f4 = Conv3D(base_filters*4, (3, 3, 3), padding='same', strides=(1, 1, 1), kernel_regularizer=l2(reg))(f4)
f4 = BatchNormalization(axis=-1)(f4)
f4 = Activation('relu')(f4)
# Upsampling stages for F3
# U1
f3 = BatchNormalization(axis=-1)(f3)
f3 = Activation('relu')(f3)
f3 = UpSampling3D(size=(2, 2, 2), name='F3_U1')(f3)
# U2
f3 = Conv3D(base_filters*4, (3, 3, 3), padding='same', strides=(1, 1, 1), kernel_regularizer=l2(reg))(f3)
f3 = BatchNormalization(axis=-1)(f3)
f3 = Activation('relu')(f3)
f3 = UpSampling3D(size=(2, 2, 2), name='F3_U2')(f3)
# Prepare
f3 = Conv3D(base_filters*4, (3, 3, 3), padding='same', strides=(1, 1, 1), kernel_regularizer=l2(reg))(f3)
f3 = BatchNormalization(axis=-1)(f3)
f3 = Activation('relu')(f3)
# Upsampling stages for F2
# U1
f2 = BatchNormalization(axis=-1)(f2)
f2 = Activation('relu')(f2)
f2 = UpSampling3D(size=(2, 2, 2), name='F2_U1')(f2)
# Prepare
f2 = Conv3D(base_filters*4, (3, 3, 3), padding='same', strides=(1, 1, 1), kernel_regularizer=l2(reg))(f2)
f2 = BatchNormalization(axis=-1)(f2)
f2 = Activation('relu')(f2)
# Prepare F1
f1 = BatchNormalization(axis=-1)(f1)
f1 = Activation('relu')(f1)
f3 = Add()([f4, f3])
f2 = Add()([f3, f2])
f1 = Add()([f2, f1])
f1 = Conv3D(base_filters*4, (3, 3, 3), padding='same', strides=(1,1,1), kernel_regularizer=l2(reg))(f1)
f1 = BatchNormalization(axis=-1)(f1)
f1 = Activation('relu')(f1)
final = Conv3D(1, (3, 3, 3), activation='sigmoid', padding='same', strides=(1, 1, 1), name='Level1')(f1)
else:
sys.exit('Loss function for Panoptic FPN must be BCE, Dice, or Focal.')
elif kwargs['model'] == 'fpn':
if loss == 'bce' or loss == 'dice' or loss == 'focal':
f1 = Conv3D(1, (3, 3, 3), activation='sigmoid', padding='same', strides=(1, 1, 1), name='Level1')(f1)
f2 = Conv3D(1, (3, 3, 3), activation='sigmoid', padding='same', strides=(1, 1, 1), name='Level2')(f2)
f3 = Conv3D(1, (3, 3, 3), activation='sigmoid', padding='same', strides=(1, 1, 1), name='Level3')(f3)
f4 = Conv3D(1, (3, 3, 3), activation='sigmoid', padding='same', strides=(1, 1, 1), name='Level4')(f4)
else:
f1 = Conv3D(1, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(f1)
f2 = Conv3D(1, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(f2)
f3 = Conv3D(1, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(f3)
f4 = Conv3D(1, (3, 3, 3), activation='relu', padding='same', strides=(1, 1, 1))(f4)
model = Model(inputs=inputs, outputs=final,name='some_unique_name')
#print(model.summary())
return model
def get_attention_lstm_intra_warrant(word_index_to_embeddings_map,
max_len,
rich_context: bool = False,
**kwargs):
# converting embeddings to numpy 2d array: shape = (vocabulary_size, 300)
embeddings = np.asarray([
np.array(x, dtype=np.float32)
for x in word_index_to_embeddings_map.values()
])
print('embeddings.shape', embeddings.shape)
lstm_size = kwargs.get('lstm_size')
dropout = kwargs.get('dropout')
assert lstm_size
assert dropout
# define basic four input layers - for warrant0, warrant1, reason, claim
sequence_layer_warrant0_input = Input(shape=(max_len, ),
dtype='int32',
name="sequence_layer_warrant0_input")
sequence_layer_warrant1_input = Input(shape=(max_len, ),
dtype='int32',
name="sequence_layer_warrant1_input")
sequence_layer_reason_input = Input(shape=(max_len, ),
dtype='int32',
name="sequence_layer_reason_input")
sequence_layer_claim_input = Input(shape=(max_len, ),
dtype='int32',
name="sequence_layer_claim_input")
sequence_layer_debate_input = Input(shape=(max_len, ),
dtype='int32',
name="sequence_layer_debate_input")
# now define embedded layers of the input
word_emb_layer = Embedding(embeddings.shape[0],
embeddings.shape[1],
input_length=max_len,
weights=[embeddings],
mask_zero=True)
embedded_layer_warrant0_input = word_emb_layer(
sequence_layer_warrant0_input)
embedded_layer_warrant1_input = word_emb_layer(
sequence_layer_warrant1_input)
embedded_layer_reason_input = word_emb_layer(sequence_layer_reason_input)
embedded_layer_claim_input = word_emb_layer(sequence_layer_claim_input)
embedded_layer_debate_input = word_emb_layer(sequence_layer_debate_input)
bidi_lstm_layer_warrant0 = Bidirectional(
LSTM(lstm_size, return_sequences=True),
name='BiDiLSTM-W0')(embedded_layer_warrant0_input)
bidi_lstm_layer_warrant1 = Bidirectional(
LSTM(lstm_size, return_sequences=True),
name='BiDiLSTM-W1')(embedded_layer_warrant1_input)
bidi_lstm_layer_reason = Bidirectional(
LSTM(lstm_size, return_sequences=True),
name='BiDiLSTM-Reason')(embedded_layer_reason_input)
bidi_lstm_layer_claim = Bidirectional(
LSTM(lstm_size, return_sequences=True),
name='BiDiLSTM-Claim')(embedded_layer_claim_input)
# add context to the attention layer
bidi_lstm_layer_debate = Bidirectional(
LSTM(lstm_size, return_sequences=True),
name='BiDiLSTM-Context')(embedded_layer_debate_input)
# max-pooling
max_pool_lambda_layer = Lambda(
lambda x: keras.backend.max(x, axis=1, keepdims=False),
output_shape=lambda x: (x[0], x[2]))
max_pool_lambda_layer.supports_masking = True
# two attention vectors
if rich_context:
attention_vector_for_w0 = max_pool_lambda_layer(
concatenate([
bidi_lstm_layer_reason, bidi_lstm_layer_claim,
bidi_lstm_layer_warrant1, bidi_lstm_layer_debate
]))
attention_vector_for_w1 = max_pool_lambda_layer(
concatenate([
bidi_lstm_layer_reason, bidi_lstm_layer_claim,
bidi_lstm_layer_warrant0, bidi_lstm_layer_debate
]))
else:
attention_vector_for_w0 = max_pool_lambda_layer(
concatenate([
bidi_lstm_layer_reason, bidi_lstm_layer_claim,
bidi_lstm_layer_warrant1
]))
attention_vector_for_w1 = max_pool_lambda_layer(
concatenate([
bidi_lstm_layer_reason, bidi_lstm_layer_claim,
bidi_lstm_layer_warrant0
]))
attention_warrant0 = AttentionLSTM(lstm_size)(
bidi_lstm_layer_warrant0, constants=attention_vector_for_w0)
attention_warrant1 = AttentionLSTM(lstm_size)(
bidi_lstm_layer_warrant1, constants=attention_vector_for_w1)
# concatenate them
dropout_layer = Dropout(dropout)(concatenate([
add([attention_warrant0, attention_warrant1]), attention_warrant0,
attention_warrant1
]))
# and add one extra layer with ReLU
dense1 = Dense(int(lstm_size / 2), activation='relu')(dropout_layer)
output_layer = Dense(1, activation='sigmoid')(dense1)
model = Model([
sequence_layer_warrant0_input, sequence_layer_warrant1_input,
sequence_layer_reason_input, sequence_layer_claim_input,
sequence_layer_debate_input
],
output=output_layer)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# from keras.utils.visualize_util import plot
# plot(model, show_shapes=True, to_file='/tmp/model-att.png')
# from keras.utils.visualize_util import plot
# plot(model, show_shapes=True, to_file='/tmp/attlstm.png')
return model
def seg(feature_num=128,
timesteps=256,
multi_grid_layer_n=1,
multi_grid_n=3,
input_channel=1,
prog=False,
out_class=2):
layer_out = []
input_score = Input(shape=(timesteps, feature_num, input_channel),
name="input_score_48")
en = Conv2D(2**5, (7, 7), strides=(1, 1), padding="same")(input_score)
layer_out.append(en)
en_l1 = conv_block(en, 2**5, (3, 3), strides=(2, 2))
en_l1 = conv_block(en_l1, 2**5, (3, 3), strides=(1, 1))
layer_out.append(en_l1)
en_l2 = conv_block(en_l1, 2**6, (3, 3), strides=(2, 2))
en_l2 = conv_block(en_l2, 2**6, (3, 3), strides=(1, 1))
en_l2 = conv_block(en_l2, 2**6, (3, 3), strides=(1, 1))
layer_out.append(en_l2)
en_l3 = conv_block(en_l2, 2**7, (3, 3), strides=(2, 2))
en_l3 = conv_block(en_l3, 2**7, (3, 3), strides=(1, 1))
en_l3 = conv_block(en_l3, 2**7, (3, 3), strides=(1, 1))
en_l3 = conv_block(en_l3, 2**7, (3, 3), strides=(1, 1))
layer_out.append(en_l3)
en_l4 = conv_block(en_l3, 2**8, (3, 3), strides=(2, 2))
en_l4 = conv_block(en_l4, 2**8, (3, 3), strides=(1, 1))
en_l4 = conv_block(en_l4, 2**8, (3, 3), strides=(1, 1))
en_l4 = conv_block(en_l4, 2**8, (3, 3), strides=(1, 1))
en_l4 = conv_block(en_l4, 2**8, (3, 3), strides=(1, 1))
layer_out.append(en_l4)
feature = en_l4
for i in range(multi_grid_layer_n):
feature = BatchNormalization()(Activation("relu")(feature))
feature = Dropout(0.3)(feature)
m = BatchNormalization()(Conv2D(2**9, (1, 1),
strides=(1, 1),
padding="same",
activation="relu")(feature))
multi_grid = m
for ii in range(multi_grid_n):
m = BatchNormalization()(Conv2D(2**9, (3, 3),
strides=(1, 1),
dilation_rate=2**ii,
padding="same",
activation="relu")(feature))
multi_grid = concatenate([multi_grid, m])
multi_grid = Dropout(0.3)(multi_grid)
feature = Conv2D(2**9, (1, 1), strides=(1, 1),
padding="same")(multi_grid)
layer_out.append(feature)
feature = BatchNormalization()(Activation("relu")(feature))
feature = Conv2D(2**8, (1, 1), strides=(1, 1), padding="same")(feature)
feature = add([feature, en_l4])
de_l1 = transpose_conv_block(feature, 2**7, (3, 3), strides=(2, 2))
layer_out.append(de_l1)
skip = de_l1
de_l1 = BatchNormalization()(Activation("relu")(de_l1))
de_l1 = concatenate(
[de_l1, BatchNormalization()(Activation("relu")(en_l3))])
de_l1 = Dropout(0.4)(de_l1)
de_l1 = Conv2D(2**7, (1, 1), strides=(1, 1), padding="same")(de_l1)
de_l1 = add([de_l1, skip])
de_l2 = transpose_conv_block(de_l1, 2**6, (3, 3), strides=(2, 2))
layer_out.append(de_l2)
skip = de_l2
de_l2 = BatchNormalization()(Activation("relu")(de_l2))
de_l2 = concatenate(
[de_l2, BatchNormalization()(Activation("relu")(en_l2))])
de_l2 = Dropout(0.4)(de_l2)
de_l2 = Conv2D(2**6, (1, 1), strides=(1, 1), padding="same")(de_l2)
de_l2 = add([de_l2, skip])
de_l3 = transpose_conv_block(de_l2, 2**5, (3, 3), strides=(2, 2))
layer_out.append(de_l3)
skip = de_l3
de_l3 = BatchNormalization()(Activation("relu")(de_l3))
de_l3 = concatenate(
[de_l3, BatchNormalization()(Activation("relu")(en_l1))])
de_l3 = Dropout(0.4)(de_l3)
de_l3 = Conv2D(2**5, (1, 1), strides=(1, 1), padding="same")(de_l3)
de_l3 = add([de_l3, skip])
de_l4 = transpose_conv_block(de_l3, 2**5, (3, 3), strides=(2, 2))
layer_out.append(de_l4)
de_l4 = BatchNormalization()(Activation("relu")(de_l4))
de_l4 = Dropout(0.4)(de_l4)
out = Conv2D(out_class, (1, 1),
strides=(1, 1),
padding="same",
name='prediction')(de_l4)
if (prog):
model = Model(inputs=input_score, outputs=layer_out)
else:
model = Model(inputs=input_score, outputs=out)
return model