def fcn_8s(num_classes, input_shape, lr_init, lr_decay, vgg_weight_path=None):
img_input = Input(input_shape)
# Block 1
x = Conv2D(64, (3, 3), padding='same', name='block1_conv1')(img_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), padding='same', name='block1_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D()(x)
# Block 2
x = Conv2D(128, (3, 3), padding='same', name='block2_conv1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), padding='same', name='block2_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D()(x)
# Block 3
x = Conv2D(256, (3, 3), padding='same', name='block3_conv1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', name='block3_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', name='block3_conv3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
block_3_out = MaxPooling2D()(x)
# Block 4
x = Conv2D(512, (3, 3), padding='same', name='block4_conv1')(block_3_out)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='block4_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='block4_conv3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
block_4_out = MaxPooling2D()(x)
# Block 5
x = Conv2D(512, (3, 3), padding='same', name='block5_conv1')(block_4_out)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='block5_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(512, (3, 3), padding='same', name='block5_conv3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D()(x)
# Load pretrained weights.
if vgg_weight_path is not None:
vgg16 = Model(img_input, x)
vgg16.load_weights(vgg_weight_path, by_name=True)
# Convolutinalized fully connected layer.
x = Conv2D(4096, (7, 7), activation='relu', padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(4096, (1, 1), activation='relu', padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Classifying layers.
x = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(x)
x = BatchNormalization()(x)
block_3_out = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(block_3_out)
block_3_out = BatchNormalization()(block_3_out)
block_4_out = Conv2D(num_classes, (1, 1), strides=(1, 1), activation='linear')(block_4_out)
block_4_out = BatchNormalization()(block_4_out)
x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 2, x.shape[2] * 2)))(x)
x = Add()([x, block_4_out])
x = Activation('relu')(x)
x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 2, x.shape[2] * 2)))(x)
x = Add()([x, block_3_out])
x = Activation('relu')(x)
x = Lambda(lambda x: tf.image.resize_images(x, (x.shape[1] * 8, x.shape[2] * 8)))(x)
x = Activation('softmax')(x)
model = Model(img_input, x)
model.compile(optimizer=Adam(lr=lr_init, decay=lr_decay),
loss='categorical_crossentropy',
metrics=[dice_coef])
return model
def modelv2(nbChannels,
stride=2,
nbFilters=[64, 64, 128, 128, 256],
nbEmbeddingDim=32,
nbQuantizationBins=16,
nbResidualLayers=1,
**kwargs):
inputs = Input(shape=(None, nbChannels))
x = inputs
kernelSize = 5
with K.name_scope('encoder'):
for i, n in enumerate(nbFilters):
# Non-linear convolution followed by linear downsampling
x = Conv1D(n,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-ds-%d' % (i + 1))(x)
x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x)
# Residual blocks
for _ in range(nbResidualLayers):
fx = Conv1D(n,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same')(x)
fx = PReLU(alpha_initializer=Constant(0.25),
shared_axes=[1])(fx)
fx = Conv1D(n,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same')(fx)
x = Add()([fx, x])
x = PReLU(alpha_initializer=Constant(0.25),
shared_axes=[1])(x)
x = Resample1D(scale=1.0 / stride, method='linear')(x)
x = Conv1D(nbEmbeddingDim,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-embedding')(x)
x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x)
# Quantization
if nbQuantizationBins is not None:
quantization = SoftmaxQuantization(nbQuantizationBins,
name='quantization')
x = quantization(x)
# Dequantization
if nbQuantizationBins is not None:
x = SoftmaxDequantization(nbQuantizationBins)(x)
kernelSize = 5
with K.name_scope('decoder'):
# Upsampling with resize convolutions
for i, n in enumerate(reversed(nbFilters)):
# Linear upsampling followed by non-linear convolution
x = Resample1D(stride, method='linear')(x)
x = Conv1D(n,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-us-%d' % (i + 1))(x)
x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x)
# Residual blocks
for _ in range(nbResidualLayers):
fx = Conv1D(n,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same')(x)
fx = PReLU(alpha_initializer=Constant(0.25),
shared_axes=[1])(fx)
fx = Conv1D(n,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same')(fx)
x = Add()([fx, x])
x = PReLU(alpha_initializer=Constant(0.25),
shared_axes=[1])(x)
xr = Conv1D(nbChannels,
kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-out')(x)
outputs = xr
return Model(inputs, outputs, **kwargs)
def generate_model(weight_decay=0.0005):
'''
define the model structure
---------------------------------------------------------------------------
INPUT:
weight_decay: all the weights in the layer would be decayed by this factor
OUTPUT:
model: the model structure after being defined
# References
- [An Improved Deep Learning Architecture for Person Re-Identification]
---------------------------------------------------------------------------
'''
def upsample_neighbor_function(input_x):
#向input_x填充八个0值
input_x_pad = K.spatial_2d_padding(input_x, padding=((2,2),(2,2)))
x_length = K.int_shape(input_x)[1]
y_length = K.int_shape(input_x)[2]
output_x_list = []
output_y_list = []
for i_x in range(2, x_length + 2):
for i_y in range(2, y_length + 2):
output_y_list.append(input_x_pad[:,i_x-2:i_x+3,i_y-2:i_y+3,:])
output_x_list.append(K.concatenate(output_y_list, axis=2))
output_y_list = []
return K.concatenate(output_x_list, axis=1)
max_pooling = MaxPooling2D()
x1_input = Input(shape=(160,60,3))
x2_input = Input(shape=(160,60,3))
#kernel_regularizer: 卷积核化的正则化，可选.
share_conv_1 = Conv2D(20, 5, kernel_regularizer=l2(weight_decay), activation="relu")
x1 = share_conv_1(x1_input)
x2 = share_conv_1(x2_input)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
share_conv_2 = Conv2D(25, 5, kernel_regularizer=l2(weight_decay), activation="relu")
x1 = share_conv_2(x1)
x2 = share_conv_2(x2)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
upsample_same = UpSampling2D(size=(5, 5))
x1_up = upsample_same(x1)
x2_up = upsample_same(x2)
upsample_neighbor = Lambda(upsample_neighbor_function)
x1_nn = upsample_neighbor(x1)
x2_nn = upsample_neighbor(x2)
negative = Lambda(lambda x: -x)
x1_nn = negative(x1_nn)
x2_nn = negative(x2_nn)
x1 = Add()([x1_up, x2_nn])
x2 = Add()([x2_up, x1_nn])
conv_3_1 = Conv2D(25, 5, strides=(5, 5), kernel_regularizer=l2(weight_decay), activation="relu")
conv_3_2 = Conv2D(25, 5, strides=(5, 5), kernel_regularizer=l2(weight_decay), activation="relu")
x1 = conv_3_1(x1)
x2 = conv_3_2(x2)
conv_4_1 = Conv2D(25, 3, kernel_regularizer=l2(weight_decay), activation="relu")
conv_4_2 = Conv2D(25, 3, kernel_regularizer=l2(weight_decay), activation="relu")
x1 = conv_4_1(x1)
x2 = conv_4_2(x2)
x1 = max_pooling(x1)
x2 = max_pooling(x2)
y = Concatenate()([x1, x2])
y = Flatten()(y)
y = Dense(500, kernel_regularizer=l2(weight_decay), activation='relu')(y)
y = Dense(2, kernel_regularizer=l2(weight_decay), activation='softmax')(y)
model = Model(inputs=[x1_input, x2_input], outputs=[y])
model.summary()
return model
def unet2D(
x_in,
input_shape,
out_im_chans,
nf_enc=[64, 64, 128, 128, 256, 256, 512],
nf_dec=None,
regularizer=None,
initializer=None,
layer_prefix='unet',
n_convs_per_stage=1,
use_residuals=False,
use_maxpool=False,
concat_at_stages=None,
do_last_conv=True,
ks=3,
):
reg_params = {}
if regularizer == 'l1':
reg = regularizers.l1(1e-6)
else:
reg = None
if initializer == 'zeros':
reg_params['kernel_initializer'] = initializers.Zeros()
x = x_in
encodings = []
for i in range(len(nf_enc)):
if not use_maxpool and i > 0:
x = LeakyReLU(0.2)(x)
for j in range(n_convs_per_stage):
if nf_enc[
i] is not None: # in case we dont want to convolve at the first resolution
x = Conv2D(nf_enc[i],
kernel_regularizer=reg,
kernel_size=ks,
strides=(1, 1),
padding='same',
name='{}_enc_conv2D_{}_{}'.format(
layer_prefix, i, j + 1))(x)
if concat_at_stages and concat_at_stages[i] is not None:
x = Concatenate(axis=-1)([x, concat_at_stages[i]])
if j == 0 and use_residuals:
residual_input = x
elif j == n_convs_per_stage - 1 and use_residuals:
x = Add()([residual_input, x])
x = LeakyReLU(0.2)(x)
if i < len(nf_enc) - 1:
encodings.append(x)
if use_maxpool:
x = MaxPooling2D(pool_size=(2, 2),
padding='same',
name='{}_enc_maxpool_{}'.format(
layer_prefix, i))(x)
else:
x = Conv2D(nf_enc[i],
kernel_size=ks,
strides=(2, 2),
padding='same',
name='{}_enc_conv2D_{}'.format(layer_prefix, i))(x)
print('Encodings to concat later: {}'.format(encodings))
if nf_dec is None:
nf_dec = list(reversed(nf_enc[1:]))
print('Decoder channels: {}'.format(nf_dec))
for i in range(len(nf_dec)):
curr_shape = x.get_shape().as_list()[1:-1]
# if we're not at full resolution, keep upsampling
if np.any(list(curr_shape[:2]) < list(input_shape[:2])):
x = UpSampling2D(size=(2, 2),
name='{}_dec_upsamp_{}'.format(layer_prefix,
i))(x)
# if we still have things to concatenate, do that
if (i + 1) <= len(encodings):
curr_shape = x.get_shape().as_list()[1:-1]
concat_with_shape = encodings[-i - 1].get_shape().as_list()[1:-1]
x = _pad_or_crop_to_shape(x, curr_shape, concat_with_shape)
x = Concatenate()([x, encodings[-i - 1]])
residual_input = x
for j in range(n_convs_per_stage):
x = Conv2D(nf_dec[i],
kernel_regularizer=reg,
kernel_size=ks,
strides=(1, 1),
padding='same',
name='{}_dec_conv2D_{}_{}'.format(layer_prefix, i,
j))(x)
if j == 0 and use_residuals:
residual_input = x
elif j == n_convs_per_stage - 1 and use_residuals:
x = Add()([residual_input, x])
x = LeakyReLU(0.2)(x)
#x = Concatenate()([x, encodings[0]])
'''
for j in range(n_convs_per_stage - 1):
x = Conv2D(out_im_chans,
kernel_regularizer=reg,
kernel_size=ks, strides=(1, 1), padding='same',
name='{}_dec_conv2D_last_{}'.format(layer_prefix, j))(x)
x = LeakyReLU(0.2)(x)
'''
if do_last_conv:
y = Conv2D(out_im_chans,
kernel_size=1,
padding='same',
kernel_regularizer=reg,
name='{}_dec_conv2D_final'.format(layer_prefix))(
x) # add your own activation after this model
else:
y = x
# add your own activation after this model
return y
def unet3D(
x_in,
img_shape,
out_im_chans,
nf_enc=[64, 64, 128, 128, 256, 256, 512],
nf_dec=None,
regularizer=None,
initializer=None,
layer_prefix='unet',
n_convs_per_stage=1,
include_residual=False,
use_maxpool=True,
max_time_downsample=None,
n_tasks=1,
use_dropout=False,
do_unpool=False,
do_last_conv=True,
):
ks = 3
if max_time_downsample is None:
max_time_downsample = len(
nf_enc) # downsample in time all the way down
encoding_im_sizes = np.asarray([(
int(np.ceil(img_shape[0] / 2.0**i)),
int(np.ceil(img_shape[1] / 2.0**i)),
int(np.ceil(img_shape[2] / 2.0**i)),
) for i in range(0,
len(nf_enc) + 1)])
else:
encoding_im_sizes = np.asarray([(
int(np.ceil(img_shape[0] / 2.0**i)),
int(np.ceil(img_shape[1] / 2.0**i)),
max(int(np.ceil(img_shape[2] / 2.0**(max_time_downsample))),
int(np.ceil(img_shape[2] / 2.0**i))),
) for i in range(0,
len(nf_enc) + 1)])
reg_params = {}
if regularizer == 'l1':
reg = regularizers.l1(1e-6)
else:
reg = None
if initializer == 'zeros':
reg_params['kernel_initializer'] = initializers.Zeros()
x = x_in
encodings = []
encoding_im_sizes = []
for i in range(len(nf_enc)):
if not use_maxpool and i > 0:
x = LeakyReLU(0.2)(x)
for j in range(n_convs_per_stage):
if nf_enc[
i] is not None: # in case we dont want to convovle at max resolution
x = Conv3D(nf_enc[i],
kernel_regularizer=reg,
kernel_size=ks,
strides=(1, 1, 1),
padding='same',
name='{}_enc_conv3D_{}_{}'.format(
layer_prefix, i, j + 1))(x)
#if use_dropout:
# x = Dropout(0.2)(x)
if j == 0 and include_residual:
residual_input = x
elif j == n_convs_per_stage - 1 and include_residual:
x = Add()([residual_input, x])
x = LeakyReLU(0.2)(x)
encodings.append(x)
encoding_im_sizes.append(np.asarray(x.get_shape().as_list()[1:-1]))
# only downsample if we haven't reached the max
if i >= max_time_downsample:
ds = (2, 2, 1)
else:
ds = (2, 2, 2)
if i < len(nf_enc) - 1:
if use_maxpool:
x = MaxPooling3D(pool_size=ds,
padding='same',
name='{}_enc_maxpool_{}'.format(
layer_prefix, i))(x)
#x, pool_idxs = Lambda(lambda x:._max_pool_3d_with_argmax(x, ksize=ks, strides=(2, 2, 2), padding='same'), name='{}_enc_maxpool3dwithargmax_{}'.format(layer_prefix, i))(x)
else:
x = Conv3D(nf_enc[i],
kernel_size=ks,
strides=ds,
padding='same',
name='{}_enc_conv3D_{}'.format(layer_prefix, i))(x)
if nf_dec is None:
nf_dec = list(reversed(nf_enc[1:]))
decoder_outputs = []
x_encoded = x
print(encoding_im_sizes)
print(nf_dec)
for ti in range(n_tasks):
decoding_im_sizes = []
x = x_encoded
for i in range(len(nf_dec)):
curr_shape = x.get_shape().as_list()[1:-1]
print('Current shape {}, img shape {}'.format(
x.get_shape().as_list(), img_shape))
# only do upsample if we are not yet at max resolution
if np.any(curr_shape < list(img_shape[:len(curr_shape)])):
# TODO: fix this for time
'''
if i < len(nf_dec) - max_time_downsample + 1 \
or curr_shape[-1] >= encoding_im_sizes[-i-2][-1]: # if we are already at the correct time scale
us = (2, 2, 1)
else:
'''
us = (2, 2, 2)
#decoding_im_sizes.append(encoding_im_sizes[-i-1] * np.asarray(us))
x = UpSampling3D(size=us,
name='{}_dec{}_upsamp_{}'.format(
layer_prefix, ti, i))(x)
# just concatenate the final layer here
if i <= len(encodings) - 2:
x = _pad_or_crop_to_shape_3D(
x, np.asarray(x.get_shape().as_list()[1:-1]),
encoding_im_sizes[-i - 2])
x = Concatenate(axis=-1)([x, encodings[-i - 2]])
#x = LeakyReLU(0.2)(x)
residual_input = x
for j in range(n_convs_per_stage):
x = Conv3D(nf_dec[i],
kernel_regularizer=reg,
kernel_size=ks,
strides=(1, 1, 1),
padding='same',
name='{}_dec{}_conv3D_{}_{}'.format(
layer_prefix, ti, i, j))(x)
if use_dropout and i < 2:
x = Dropout(0.2)(x)
if j == 0 and include_residual:
residual_input = x
elif j == n_convs_per_stage - 1 and include_residual:
x = Add()([residual_input, x])
x = LeakyReLU(0.2)(x)
if do_last_conv:
y = Conv3D(out_im_chans,
kernel_size=1,
padding='same',
kernel_regularizer=reg,
name='{}_dec{}_conv3D_final'.format(layer_prefix, ti))(
x) # add your own activation after this model
else:
y = x
decoder_outputs.append(y)
# add your own activation after this model
if n_tasks == 1:
return y
else:
return decoder_outputs
def create_deepconn_dp(self):
dotproduct = Dot(axes=1)([self.towerU, self.towerM])
output = Add()([self.outNeuron, dotproduct])
self.model = Model(inputs=[self.inputU, self.inputM], outputs=[output])
self.model.compile(optimizer='Adam', loss='mse')
def encoder3D(x,
img_shape,
conv_chans=None,
n_convs_per_stage=1,
min_h=5,
min_c=None,
prefix='vte',
ks=3,
return_skips=False,
use_residuals=False,
use_maxpool=False,
max_time_downsample=None):
skip_layers = []
concat_skip_sizes = []
if max_time_downsample is None:
# do not attempt to downsample beyond 1
max_time_downsample = int(np.floor(np.log2(img_shape[-2]))) - 1
print('Max downsamples in time: {}'.format(max_time_downsample))
if conv_chans is None:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [min_c * 2] * (n_convs - 1) + [min_c]
elif not type(conv_chans) == list:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [conv_chans] * (n_convs - 1) + [min_c]
for i in range(len(conv_chans)):
if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None:
for ci in range(n_convs_per_stage):
x = Conv3D(conv_chans[i],
kernel_size=ks,
padding='same',
name='{}_enc_conv3D_{}_{}'.format(
prefix, i, ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))(
[residual_input, x])
x = LeakyReLU(0.2,
name='{}_enc_leakyrelu_{}_{}'.format(
prefix, i, ci + 1))(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(x.get_shape().as_list()[1:-1])
# only downsample if we are below the max number of downsamples in time
if i < max_time_downsample:
strides = (2, 2, 2)
else:
strides = (2, 2, 1)
if use_maxpool:
x = MaxPooling3D(pool_size=strides,
name='{}_enc_maxpool_{}'.format(prefix, i))(x)
else:
x = Conv3D(conv_chans[i],
kernel_size=ks,
strides=strides,
padding='same',
name='{}_enc_conv3D_{}'.format(prefix, i))(x)
if i < len(conv_chans) - 1: # no activation on last convolution
x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}'.format(prefix, i))(x)
if min_c is not None and min_c > 0:
if n_convs_per_stage is not None and n_convs_per_stage > 1:
for ci in range(n_convs_per_stage):
x = Conv3D(min_c,
kernel_size=ks,
padding='same',
name='{}_enc_conv3D_last_{}'.format(prefix,
ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(
name='{}_enc_{}_add_residual'.format(prefix, 'last'))(
[residual_input, x])
x = LeakyReLU(0.2,
name='{}_enc_leakyrelu_last'.format(prefix))(x)
x = Conv3D(min_c,
kernel_size=ks,
strides=(1, 1, 1),
padding='same',
name='{}_enc_conv3D_last'.format(prefix))(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(x.get_shape().as_list()[1:-1])
if return_skips:
return x, skip_layers, concat_skip_sizes
else:
return x
def train(self, trainingdata, validation_data, pretrained=None):
in_dim = len(
ConfigurationVector(Configuration([], None), None).get_vector())
# Shared model
if pretrained:
base_model = load_pretrained_base_model(pretrained)
else:
base_model = make_base_model(in_dim)
self.base_model = base_model
# All golden decisions = ONE SEQUENCE
golden = []
golden_inputs = []
for i in range(self.k):
name = "Golden" + str(i)
# Configuration feature vector input
state_input = Input(shape=(in_dim, ), name=name)
golden_inputs.append(state_input)
# Decision input
decision_input = Input(shape=(4, ))
golden_inputs.append(decision_input)
# Probabilities of decisions under model
distribution = base_model(state_input)
# Decision activation
output = Dot(axes=0)([distribution, decision_input])
golden.append(output)
# Sum all probabilities in golden sequence
golden_sum = Add()(golden)
# All decisions in the beam = MULTIPLE SEQUENCES
beam = []
beam_inputs = []
for i in range(self.b):
sequence = []
for j in range(self.k):
name = "Beam{i}.{j}".format(i=i, j=j)
# Configuration feature vector input
state_input = Input(shape=(in_dim, ), name=name)
beam_inputs.append(state_input)
# Decision input
decision_input = Input(shape=(4, ))
beam_inputs.append(decision_input)
# Probabilities of decisions under model
distribution = base_model(state_input)
# Decision activation
output = Dot(axes=0)([distribution, decision_input])
sequence.append(output)
beam.append(sequence)
# Make the inner sums over all sequences in the beam
inner_sequence_sums = list(map(Add(), beam))
# Apply exp to all inner sums
exp_activation = Activation(K.exp)
inner_sequence_sums = list(map(exp_activation, inner_sequence_sums))
if len(inner_sequence_sums) > 1:
# Sum all beam sequences together
outer_sum = Add()(inner_sequence_sums)
else:
outer_sum = inner_sequence_sums
# Log activation
outer_sum = Activation(K.log)(outer_sum)
# Negate golden sum
golden_sum = Activation(negativeActivation)(golden_sum)
output = Add()([golden_sum, outer_sum])
# Add beam inputs to all inputs
inputs = golden_inputs + beam_inputs
model = Model(inputs, output)
self.machine = model
self.graph = tf.get_default_graph()
model.compile(loss=global_norm_loss, optimizer=SGD())
model.fit_generator(
self.generate_training_data(trainingdata),
verbose=1,
epochs=100,
steps_per_epoch=20,
callbacks=[earlyStopping, csv_logger],
validation_data=self.generate_training_data(validation_data),
validation_steps=20,
max_q_size=1)
self.save()
def segnet_crf_res(nClasses,
optimizer=None,
input_height=360,
input_width=480):
kernel = 3
filter_size = 64
pad = 1
pool_size = 2
img_input = Input(shape=(input_height, input_width, 3))
# encoder
x = ZeroPadding2D(padding=(pad, pad))(img_input)
x = Convolution2D(filter_size, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
l1 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(128, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
l2 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(256, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
l3 = x
x = MaxPooling2D(pool_size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(512, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
l4 = x
x = Activation('relu')(x)
# decoder
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(512, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l4, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(256, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l3, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(128, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l2, x])
x = UpSampling2D(size=(pool_size, pool_size))(x)
x = ZeroPadding2D(padding=(pad, pad))(x)
x = Convolution2D(filter_size, (kernel, kernel), padding='valid')(x)
x = BatchNormalization()(x)
x = Add()([l1, x])
x = Convolution2D(nClasses, (1, 1), padding='valid')(x)
out = CrfRnnLayer(image_dims=(input_height, input_width),
num_classes=nClasses,
theta_alpha=160.,
theta_beta=3.,
theta_gamma=3.,
num_iterations=5,
name='crfrnn')([x, img_input])
# out = x
a = Model(inputs=img_input, outputs=out)
model = []
a.outputHeight = a.output_shape[1]
a.outputWidth = a.output_shape[2]
out = Reshape((a.outputHeight * a.outputWidth, nClasses),
input_shape=(nClasses, a.outputHeight, a.outputWidth))(out)
out = Activation('softmax')(out)
# if not optimizer is None:
# model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy'])
model = Model(inputs=img_input, outputs=out)
model.outputHeight = a.outputHeight
model.outputWidth = a.outputWidth
return model
def gossip_conv_block(self, images, masks, nfilters):
cfs = self.conv_filter_size
conv = lambda: Conv2D(nfilters,
cfs,
activation='linear',
padding=self.conv_padding,
kernel_initializer=glorot_uniform(seed=42))
if self.shared:
cc = conv()
conv = lambda: cc
images = [conv()(l) for l in images]
"""
constant = Constant(value=1. / float(cfs[0] * cfs[1]))
mask_conv = Conv2D(1, cfs, activation='linear',
padding=self.conv_padding,
kernel_initializer=constant,
bias_initializer=Zeros(),
trainable=False)
mask_conv.trainable = False
masks = [mask_conv(l) for l in masks]
"""
if self.conv_padding == 'valid':
crop_cfs = int(floor(cfs[0] / 2))
crop_cfs = (crop_cfs, crop_cfs)
crop = Cropping2D(cropping=(crop_cfs, crop_cfs))
masks = [crop(m) for m in masks]
# Merge connections
act_diff = [
Lambda(lambda_diff)([this_, other_])
for this_, other_ in zip(images, images[::-1])
]
#this_high = [Activation('relu')(d) for d in act_diff]
other_high = [
Activation('relu')(Lambda(lambda_neg)(d)) for d in act_diff
]
# FIXME: without relu
#other_high = [Lambda(lambda_neg)(d) for d in act_diff]
stimuli = [
Lambda(lambda_neg)(Multiply()([m, d]))
for m, d in zip(masks, other_high)
]
"""
stimuli = [Lambda(lambda_diff)([p, n])
for p, n in zip(pos_stimuli, neg_stimuli)]
#stimuli = Lambda(lambda_diff)(stimuli)
#stimuli = [stimuli, Lambda(lambda_neg)(stimuli)]
"""
#stimuli = [Lambda(lambda x: self.reciprocal_stimuli * x)(s)
#for s in stimuli]
# Second merge
images = [Add()([i, s]) for i, s in zip(images, stimuli)]
#images = [Activation('relu')(i) for i in images]
return images, masks
def resblock(x, kernel_size, resample, nfilters, norm=BatchNormalization):
assert resample in ["UP", "SAME", "DOWN"]
if resample == "UP":
shortcut = UpSampling2D(size=(2, 2))(x)
shortcut = Conv2D(nfilters,
kernel_size,
padding='same',
kernel_initializer='he_uniform',
use_bias=True)(shortcut)
convpath = norm()(x)
convpath = Activation('relu')(convpath)
convpath = UpSampling2D(size=(2, 2))(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=False,
padding='same')(convpath)
convpath = norm()(convpath)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=True,
padding='same')(convpath)
y = Add()([shortcut, convpath])
elif resample == "SAME":
shortcut = Conv2D(nfilters,
kernel_size,
padding='same',
kernel_initializer='he_uniform',
use_bias=True)(x)
convpath = norm()(x)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=False,
padding='same')(convpath)
convpath = norm()(convpath)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=True,
padding='same')(convpath)
y = Add()([shortcut, convpath])
else:
shortcut = AveragePooling2D(pool_size=(2, 2))(x)
shortcut = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
padding='same',
use_bias=True)(shortcut)
convpath = norm()(x)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=False,
padding='same')(convpath)
convpath = AveragePooling2D(pool_size=(2, 2))(convpath)
convpath = norm()(convpath)
convpath = Activation('relu')(convpath)
convpath = Conv2D(nfilters,
kernel_size,
kernel_initializer='he_uniform',
use_bias=True,
padding='same')(convpath)
y = Add()([shortcut, convpath])
return y
def separable_residual_block(inputs,
n_filters_list=[256, 256, 256],
block_id="entry_block2",
skip_type="sum",
stride=1,
rate=1,
weight_decay=1e-4,
kernel_initializer="he_normal",
bn_epsilon=1e-3,
bn_momentum=0.99):
""" separable residual block
:param inputs: 4-D tensor, shape of (batch_size, height, width, channel).
:param n_filters_list: list of int, numbers of filters in the separable convolutions, default [256, 256, 256].
:param block_id: string, default "entry_block2".
:param skip_type: string, one of {"sum", "conv", "none"}, default "sum".
:param stride: int, default 1.
:param rate: int, default 1.
:param weight_decay: float, default 1e-4.
:param kernel_initializer: string, default "he_normal".
:param bn_epsilon: float, default 1e-3.
:param bn_momentum: float, default 0.99.
:return: 4-D tensor, shape of (batch_size, height, width, channel).
"""
x = Activation("relu", name=block_id + "sepconv1_act")(inputs)
x = SeparableConv2D(n_filters_list[0], (3, 3),
padding='same',
use_bias=False,
name=block_id + '_sepconv1',
dilation_rate=rate,
kernel_initializer=kernel_initializer,
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name=block_id + '_sepconv1_bn',
epsilon=bn_epsilon,
momentum=bn_momentum)(x)
x = Activation('relu', name=block_id + '_sepconv2_act')(x)
x = SeparableConv2D(n_filters_list[1], (3, 3),
padding='same',
use_bias=False,
name=block_id + '_sepconv2',
dilation_rate=rate,
kernel_initializer=kernel_initializer,
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name=block_id + '_sepconv2_bn',
epsilon=bn_epsilon,
momentum=bn_momentum)(x)
x = Activation("relu", name=block_id + "_sepconv3_act")(x)
x = SeparableConv2D(n_filters_list[2], (3, 3),
padding="same",
use_bias=False,
strides=stride,
name=block_id + "_sepconv3",
dilation_rate=rate,
kernel_initializer=kernel_initializer,
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name=block_id + "_sepconv3_bn",
epsilon=bn_epsilon,
momentum=bn_momentum)(x)
if skip_type == "sum":
x = Add()([inputs, x])
elif skip_type == "conv":
shortcut = Conv2D(n_filters_list[2], (1, 1),
strides=stride,
padding='same',
use_bias=False,
kernel_initializer=kernel_initializer,
kernel_regularizer=l2(weight_decay))(inputs)
shortcut = BatchNormalization(epsilon=bn_epsilon,
momentum=bn_momentum)(shortcut)
x = Add()([shortcut, x])
else:
x = x
return x
def stresnet(c_conf=(3, 2, 32, 32), p_conf=(3, 2, 32, 32), t_conf=(3, 2, 32, 32), external_dim=8, nb_residual_unit=3):
'''
C - Temporal Closeness
P - Period
T - Trend
conf = (len_seq, nb_flow, map_height, map_width)
external_dim
'''
# main input
main_inputs = []
outputs = []
# c_conf, p_conf, t_conf
# for conf in [ c_conf]:
# if conf is not None:
# # base 模型
# len_seq, nb_flow, map_height, map_width = conf
# input = Input(shape=(nb_flow * len_seq, map_height, map_width))
# main_inputs.append(input)
# # Conv1
# # conv1 = Convolution2D(nb_filter=64, nb_row=3, nb_col=3, border_mode="same")(input)
# conv1 = Conv2D(padding="same", kernel_size=(stride, stride), filters=64)(input)
# # [nb_residual_unit] Residual Units
# residual_output = ResUnits(_residual_unit, nb_filter=64,
# repetations=nb_residual_unit,)(conv1)
# # Conv2
# activation = Activation('relu')(residual_output)
# # conv2 = Convolution2D(
# # nb_filter=nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation)
# conv2 = Convolution2D(
# nb_filter=nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation)
# outputs.append(conv2)
for conf in [c_conf]:
if conf is not None:
# base 模型
# len_seq, nb_flow, map_height, map_width = conf
# input = Input(shape=(nb_flow * len_seq, map_height, map_width))
# main_inputs.append(input)
# # Conv1
# # conv1 = Convolution2D(nb_filter=64, nb_row=3, nb_col=3, border_mode="same")(input)
# conv1 = Conv2D(padding="same", kernel_size=(3, 3), filters=64)(input)
# # [nb_residual_unit] Residual Units
# residual_output = ResUnits(_residual_unit, nb_filter=64,
# repetations=nb_residual_unit)(conv1)
# # Conv2
# activation = Activation('relu')(residual_output)
# # conv2 = Convolution2D(
# # nb_filter=nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation)
# conv2 = Convolution2D(
# nb_filter=output_nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation)
# outputs.append(conv2)
# 我的模型
len_seq, nb_flow, map_height, map_width = conf
# input=main_inputs[0]
input = Input(shape=(nb_flow * len_seq, map_height, map_width))
# main_inputs[1:3]=main_inputs[0:2]
# main_inputs[0]=input
main_inputs.append(input)
input = Reshape((len_seq, nb_flow, map_height, map_width))(input)
timeSliceOutputs = []
print(input.shape[1])
# 定义共用的CNN模型
# conv1=Reshape((nb_flow, map_height, map_width))(input[:,timeSlice])
# Conv1
aa = Conv2D(padding="same", kernel_size=(stride, stride), filters=64)
# conv1 = Conv2D(padding="same", kernel_size=(3, 3), filters=64)
# [nb_residual_unit] Residual Units
nb_residual_unit = 5
resIntput = Input(shape=(64, map_height, map_width))
# resIntput=Input(shape=aa.output_shape)
bb = ResUnits(_residual_unit, nb_filter=64,
repetations=nb_residual_unit, pool=True)(resIntput)
resModel = Model(resIntput, bb)
resModel.summary()
plot_model(resModel, to_file='resModel.png')
# Conv2
cc = Activation('relu')
# conv2 = Convolution2D(
# nb_filter=nb_flow, nb_row=3, nb_col=3, border_mode="same")(activation)
dd = Convolution2D(
nb_filter=128, nb_row=2, nb_col=2, border_mode="same")
ee = Reshape((1, -1))
for timeSlice in range(len_seq):
ii = sliceLayer(1, timeSlice)(input)
# ii=input[:, timeSlice]
conv2 = ee(dd(cc(resModel(aa(ii)))))
timeSliceOutputs.append(conv2)
convOutput = Concatenate(axis=1)(timeSliceOutputs)
# lstm = GRU(output_nb_flow * map_height * map_width)(convOutput)
# lstm = GRU(2 * map_height * map_width)(convOutput)
lstm = LSTM(600)(convOutput)
# input+lstm
out1 = sliceLayer(1, len_seq - 1)(input)
def antirectifier(x):
# x=np.reshape(x,[-1,18,2,64,64])
# x = tf.reshape(x, [-1, 18, 2, 64, 64])
# x = tf.reduce_sum(x, axis=1)
x = K.reshape(x, [-1, 18, 2, map_height, map_width])
x = K.sum(x, axis=1)
return x
def antirectifier_output_shape(input_shape):
return (input_shape[0], 2, map_height, map_width)
# out1 = Lambda(antirectifier,
# output_shape=antirectifier_output_shape)(out1)
# out2 = Flatten()(lstm)
sig = Dense(output_dim=nb_flow * map_height * map_width, activation='sigmoid')(lstm)
sig = Reshape((nb_flow, map_height, map_width))(sig)
tan = Dense(output_dim=nb_flow * map_height * map_width, activation='tanh')(lstm)
tan = Reshape((nb_flow, map_height, map_width))(tan)
den = Dense(output_dim=nb_flow * map_height * map_width)(lstm)
den = Reshape((nb_flow, map_height, map_width))(den)
# out2=Add()([Multiply()([outputs[0],sig]),tan,out1])
mul=Multiply()([out1, sig])
out2 = Add()([mul, den])
# output = Add()([out1, tan])
# outputs[0]=out2
outputs.append(out2)
# len_seq, nb_flow, map_height, map_width = c_conf
# input = main_inputs[0]
# # input = Input(shape=(nb_flow * len_seq, map_height, map_width))
# input2 = Reshape((len_seq, nb_flow * map_height * map_width))(input)
# lstm = LSTM(output_nb_flow * map_height * map_width)(input)
# output = Reshape((output_nb_flow, map_height, map_width))(lstm)
# outputs.append(output)
# outputs=outputs[1:]
# parameter-matrix-based fusion
if len(outputs) == 1:
main_output = outputs[0]
else:
new_outputs = []
for output in outputs:
# new_outputs.append(iLayer()(output))
new_outputs.append(mulLayer()(output))
main_output = Add()(new_outputs)
# main_output=Add()(outputs)
# fusing with external component
# if external_dim != None and external_dim > 0:
# # external input
# external_input = Input(shape=(external_dim,))
# main_inputs.append(external_input)
# embedding = Dense(output_dim=10)(external_input)
# embedding = Activation('relu')(embedding)
# # h1 = Dense(output_dim=nb_flow * map_height * map_width)(embedding)
# h1 = Dense(output_dim=nb_flow * map_height * map_width)(embedding)
# activation = Activation('relu')(h1)
# # external_output = Reshape((nb_flow, map_height, map_width))(activation)
# external_output = Reshape((nb_flow, map_height, map_width))(activation)
# main_output = merge([main_output, external_output], mode='sum')
# else:
# print('external_dim:', external_dim)
main_output = Activation('tanh')(main_output)
def antirectifier2(x):
#x=np.reshape(x,[-1,18,2,64,64])
x = K.reshape(x,[-1,18,2,map_height,map_width])
x = K.sum(x, axis=1)
return x
# def antirectifier_output_shape(input_shape):
#
# return (input_shape[0],2,64,64)
#
# main_output=Lambda(antirectifier2 )(main_output)
model = Model(input=main_inputs, output=main_output)
return model
def _shortcut(input, residual):
# return merge([input, residual], mode='sum')
return Add()([input, residual])
def encoder(x, img_shape,
conv_chans=None,
n_convs_per_stage=1,
min_h=5, min_c=None,
prefix='',
ks=3,
return_skips=False, use_residuals=False, use_maxpool=False, use_batchnorm=False):
skip_layers = []
concat_skip_sizes = []
n_dims = len(img_shape) - 1 # assume img_shape includes spatial dims, followed by channels
if conv_chans is None:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [min_c * 2] * (n_convs - 1) + [min_c]
elif not type(conv_chans) == list:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [conv_chans] * (n_convs - 1) + [min_c]
for i in range(len(conv_chans)):
#if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None:
for ci in range(n_convs_per_stage):
x = myConv(nf=conv_chans[i], ks=ks, strides=1, n_dims=n_dims,
prefix='{}_enc'.format(prefix),
suffix='{}_{}'.format(i, ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))([residual_input, x])
if use_batchnorm:
x = BatchNormalization()(x)
x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}_{}'.format(prefix, i, ci + 1))(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1]))
if use_maxpool and i < len(conv_chans) - 1:
# changed 5/30/19, don't pool after our last conv
x = myPool(n_dims=n_dims, prefix=prefix, suffix=i)(x)
else:
x = myConv(conv_chans[i], ks=ks, strides=2, n_dims=n_dims,
prefix='{}_enc'.format(prefix), suffix=i)(x)
# don't activate right after a maxpool, it makes no sense
if i < len(conv_chans) - 1: # no activation on last convolution
x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}'.format(prefix, i))(x)
if min_c is not None and min_c > 0:
# if the last number of channels is specified, convolve to that
if n_convs_per_stage is not None and n_convs_per_stage > 1:
for ci in range(n_convs_per_stage):
x = myConv(min_c, ks=ks, n_dims=n_dims, strides=1,
prefix='{}_enc'.format(prefix), suffix='last_{}'.format(ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_enc_{}_add_residual'.format(prefix, 'last'))([residual_input, x])
x = LeakyReLU(0.2, name='{}_enc_leakyrelu_last'.format(prefix))(x)
x = myConv(min_c, ks=ks, strides=1, n_dims=n_dims,
prefix='{}_enc'.format(prefix),
suffix='_last')(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1]))
if return_skips:
return x, skip_layers, concat_skip_sizes
else:
return x
def get_DPCNN(embedding_matrix, sequence_length, dropout_rate, recurrent_units, dense_size,
filter_sizes=3, repeat=4):
input_layer = Input(shape=(sequence_length,))
num_filters = embedding_matrix.shape[1]
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=False)(input_layer)
z = SpatialDropout1D(rate=dropout_rate)(embedding_layer)
# x = Bidirectional(CuDNNGRU(recurrent_units, return_sequences=True))(x)
conv = Convolution1D(filters=num_filters,
kernel_size=filter_sizes,
padding="same",
activation="relu",
strides=1,
kernel_regularizer=regularizers.l2(0.0001),
activity_regularizer=regularizers.l2(0.0001),
bias_regularizer=regularizers.l2(0.0001)
)(z)
conv = Convolution1D(filters=num_filters,
kernel_size=filter_sizes,
padding="same",
activation="relu",
strides=1,
kernel_regularizer=regularizers.l2(0.0001),
activity_regularizer=regularizers.l2(0.0001),
bias_regularizer=regularizers.l2(0.0001)
)(conv)
# conv = Dropout(dropout_rate)(conv)
conv = Add()([conv, z])
for i in range(repeat):
pool = MaxPooling1D(pool_size=3, strides=2, padding="same")(conv)
# conv = SpatialDropout1D(rate=dropout_rate)(pool)
conv = Convolution1D(filters=num_filters,
kernel_size=filter_sizes,
padding="same",
activation="relu",
strides=1,
kernel_regularizer=regularizers.l2(0.0001),
activity_regularizer=regularizers.l2(0.0001),
bias_regularizer=regularizers.l2(0.0001)
)(pool)
conv = Convolution1D(filters=num_filters,
kernel_size=filter_sizes,
padding="same",
activation="relu",
strides=1,
kernel_regularizer=regularizers.l2(0.0001),
activity_regularizer=regularizers.l2(0.0001),
bias_regularizer=regularizers.l2(0.0001)
)(conv)
# conv = Dropout(dropout_rate)(conv)
conv = Add()([conv, pool])
#z1 = GlobalMaxPooling1D()(conv)
#z2 = GlobalAveragePooling1D()(conv)
#z = Concatenate()([z1, z2])
z = MaxPooling1D(3)(conv)
z = Flatten()(z)
z = Dense(dense_size, activation="relu", kernel_regularizer=regularizers.l2(0.0001),
activity_regularizer=regularizers.l2(0.0001),
bias_regularizer=regularizers.l2(0.0001))(z)
# z = Dropout(0.5)(z)
output_layer = Dense(6, activation="sigmoid", kernel_regularizer=regularizers.l2(0.0001),
activity_regularizer=regularizers.l2(0.0001),
bias_regularizer=regularizers.l2(0.0001))(z)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='binary_crossentropy',
optimizer=Nadam(),
metrics=['accuracy'])
return model
def net_module(input_shape, num_outputs):
"""Builds a net architecture.
Args:
input_shape: The input shape in the form (nb_rows, nb_cols, nb_channels)
num_outputs: The number of outputs at final softmax layer
Returns:
The keras `Model`.
"""
CHANNEL_AXIS = 3
handle_dim_ordering()
if len(input_shape) != 3:
raise Exception(
"Input shape should be a tuple (nb_rows, nb_cols, nb_channels)")
# Permute dimension order if necessary
if K.image_dim_ordering() != 'tf':
input_shape = (input_shape[2], input_shape[0], input_shape[1])
input_img0 = Input(shape=input_shape, name="input_img0")
input_img1 = Input(shape=input_shape, name="input_img1")
concatenate = Concatenate(axis=CHANNEL_AXIS,
name="concatenate")([input_img0, input_img1])
base_channel = 24
block_conv_1 = conv_bn_leakyrelu_repetition_block(
filters=1 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="conv_block1")(concatenate)
block_pool_2 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid',
data_format=None,
name="pool_block2")(block_conv_1)
block_conv_2 = conv_bn_leakyrelu_repetition_block(
filters=2 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="conv_block2")(block_pool_2)
block_pool_4 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid',
data_format=None,
name="pool_block4")(block_conv_2)
block_conv_4 = conv_bn_leakyrelu_repetition_block(
filters=4 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="conv_block4")(block_pool_4)
block_pool_8 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid',
data_format=None,
name="pool_block8")(block_conv_4)
block_conv_8 = conv_bn_leakyrelu_repetition_block(
filters=8 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="conv_block8")(block_pool_8)
block_pool_16 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid',
data_format=None,
name="pool_block16")(block_conv_8)
block_conv_16 = conv_bn_leakyrelu_repetition_block(
filters=16 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="conv_block16")(block_pool_16)
block_up_8 = UpSampling2D(size=(2, 2), name="up_block8")(block_conv_16)
block_concat_8 = Concatenate(axis=CHANNEL_AXIS,
name="concat8")([block_up_8, block_conv_8])
block_expan_conv_8 = conv_bn_leakyrelu_repetition_block(
filters=8 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="expan_conv_block8")(block_concat_8)
block_up_4 = UpSampling2D(size=(2, 2),
name="up_block4")(block_expan_conv_8)
block_concat_4 = Concatenate(axis=CHANNEL_AXIS,
name="concat4")([block_up_4, block_conv_4])
block_expan_conv_4 = conv_bn_leakyrelu_repetition_block(
filters=4 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="expan_conv_block4")(block_concat_4)
block_up_2 = UpSampling2D(size=(2, 2),
name="up_block2")(block_expan_conv_4)
block_concat_2 = Concatenate(axis=CHANNEL_AXIS,
name="concat2")([block_up_2, block_conv_2])
block_expan_conv_2 = conv_bn_leakyrelu_repetition_block(
filters=2 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="expan_conv_block2")(block_concat_2)
block_up_1 = UpSampling2D(size=(2, 2),
name="up_block1")(block_expan_conv_2)
block_concat_1 = Concatenate(axis=CHANNEL_AXIS,
name="concat1")([block_up_1, block_conv_1])
block_expan_conv_1 = conv_bn_leakyrelu_repetition_block(
filters=1 * base_channel,
kernel_size=(3, 3),
repetitions=2,
first_layer_down_size=False,
alpha=0.0,
name="expan_conv_block1")(block_concat_1)
block_seg_4 = Conv2D(filters=num_outputs,
kernel_size=(1, 1),
strides=(1, 1),
padding="same",
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer="he_normal",
bias_initializer="zeros",
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
name="seg_block4")(block_expan_conv_4)
block_seg_2 = Conv2D(filters=num_outputs,
kernel_size=(1, 1),
strides=(1, 1),
padding="same",
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer="he_normal",
bias_initializer="zeros",
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
name="seg_block2")(block_expan_conv_2)
block_seg_1 = Conv2D(filters=num_outputs,
kernel_size=(1, 1),
strides=(1, 1),
padding="same",
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer="he_normal",
bias_initializer="zeros",
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
name="seg_block1")(block_expan_conv_1)
block_seg_up_2 = UpSampling2D(size=(2, 2),
name="seg_up_block2")(block_seg_4)
block_add_2 = Add(name="add_block2")([block_seg_up_2, block_seg_2])
block_seg_up_1 = UpSampling2D(size=(2, 2),
name="seg_up_block1")(block_add_2)
output = Add(name="output")([block_seg_up_1, block_seg_1])
model = Model(inputs=[input_img0, input_img1], outputs=output)
return model
wr = K.variable(value=0.25, name="wr")
for j in range(MAX_SENTENCE):
hj = Lambda((lambda X: X[:, j, :]), output_shape=(2 * HIDDEN_SIZE, ))(h)
content = Dense(1, activation='sigmoid', name="c" + str(j))(hj)
salience = Dot(axes=1, normalize=True)([d, hj])
salience = Activation('sigmoid')(salience)
weighted_content = Lambda(lambda X: wc * X)(content)
weighted_salience = Lambda(lambda X: ws * X)(salience)
if j == 0:
sj = Lambda((lambda X: 0 * X), output_shape=(2 * HIDDEN_SIZE, ))(hj)
redundancy = Dot(axes=1, normalize=True)([hj, sj])
redundancy = Activation('sigmoid')(redundancy)
weighted_redundancy = Lambda(lambda X: ws * X)(redundancy)
weighted_redundancy = Lambda(lambda X: -1 * X)(redundancy)
score = Add()(
[weighted_content, weighted_salience, weighted_redundancy])
p = Dense(2, activation='softmax', name="p" + str(j))(score)
prob = Reshape((1, 2))(p)
else:
hj_minus_one = Lambda((lambda X: X[:, j - 1, :]),
output_shape=(2 * HIDDEN_SIZE, ))(h)
pi_minus_one = Lambda((lambda X: X[:, j - 1, 1]),
output_shape=(1, ))(prob)
pi_minus_one = Reshape((1, ))(pi_minus_one)
sj_minus_one = Lambda(lambda X: pi_minus_one * X)(hj_minus_one)
sj = Add()([sj_minus_one, sj])
redundancy = Dot(axes=1, normalize=True)([hj, sj])
redundancy = Activation('sigmoid')(redundancy)
weighted_redundancy = Lambda(lambda X: ws * X)(redundancy)
weighted_redundancy = Lambda((lambda X: -1 * X))(redundancy)
score = Add()(
def encoder(x,
img_shape,
conv_chans=None,
n_convs_per_stage=1,
min_h=5,
min_c=None,
prefix='',
ks=3,
return_skips=False,
use_residuals=False,
use_maxpool=False,
use_batchnorm=False,
kernel_initializer=None,
bias_initializer=None):
skip_layers = []
concat_skip_sizes = []
n_dims = len(
img_shape
) - 1 # assume img_shape includes spatial dims, followed by channels
if conv_chans is None:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [min_c * 2] * (n_convs - 1) + [min_c]
elif not type(conv_chans) == list:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [conv_chans] * (n_convs - 1) + [min_c]
else:
n_convs = len(conv_chans)
if isinstance(ks, list):
assert len(ks) == (n_convs + 1
) # specify for each conv, as well as the last one
else:
ks = [ks] * (n_convs + 1)
for i in range(len(conv_chans)):
#if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None:
for ci in range(n_convs_per_stage):
x = myConv(nf=conv_chans[i],
ks=ks[i],
strides=1,
n_dims=n_dims,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
prefix='{}_enc'.format(prefix),
suffix='{}_{}'.format(i, ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))(
[residual_input, x])
if use_batchnorm:
x = BatchNormalization()(x)
x = LeakyReLU(0.2,
name='{}_enc_leakyrelu_{}_{}'.format(
prefix, i, ci + 1))(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1]))
if use_maxpool and i < len(conv_chans) - 1:
# changed 5/30/19, don't pool after our last conv
x = myPool(n_dims=n_dims, prefix=prefix, suffix=i)(x)
else:
x = myConv(conv_chans[i],
ks=ks[i],
strides=2,
n_dims=n_dims,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
prefix='{}_enc'.format(prefix),
suffix=i)(x)
# don't activate right after a maxpool, it makes no sense
if i < len(conv_chans) - 1: # no activation on last convolution
x = LeakyReLU(0.2,
name='{}_enc_leakyrelu_{}'.format(prefix, i))(x)
if min_c is not None and min_c > 0:
# if the last number of channels is specified, convolve to that
if n_convs_per_stage is not None and n_convs_per_stage > 1:
for ci in range(n_convs_per_stage):
# TODO: we might not have enough ks for this
x = myConv(min_c,
ks=ks[-1],
n_dims=n_dims,
strides=1,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
prefix='{}_enc'.format(prefix),
suffix='last_{}'.format(ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(
name='{}_enc_{}_add_residual'.format(prefix, 'last'))(
[residual_input, x])
x = LeakyReLU(0.2,
name='{}_enc_leakyrelu_last'.format(prefix))(x)
x = myConv(min_c,
ks=ks[-1],
strides=1,
n_dims=n_dims,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
prefix='{}_enc'.format(prefix),
suffix='_last')(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(np.asarray(x.get_shape().as_list()[1:-1]))
if return_skips:
return x, skip_layers, concat_skip_sizes
else:
return x
def resunet(shape, n_filters):
# Convolutional block: BN -> ReLU -> Conv3x3
def conv_block(inputs,
n_filters,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
batch_norm=True,
padding='same'):
if batch_norm:
x = BatchNormalization()(inputs)
else:
x = inputs
if activation:
x = Activation('relu')(x)
x = Conv2D(filters=n_filters,
kernel_size=kernel_size,
strides=strides,
padding=padding)(x)
return x
inputs = Input((*shape, 1))
# Contracting path
short1 = inputs
conv1 = conv_block(inputs, n_filters, activation=None, batch_norm=False)
conv1 = conv_block(conv1, n_filters)
short1 = conv_block(short1, n_filters, activation=None)
conv1 = Add()([conv1, short1])
short2 = conv1
conv2 = conv_block(conv1, n_filters * 2, strides=(2, 2))
conv2 = conv_block(conv2, n_filters * 2)
short2 = conv_block(short2, n_filters * 2, strides=(2, 2), activation=None)
conv2 = Add()([conv2, short2])
short3 = conv2
conv3 = conv_block(conv2, n_filters * 4, strides=(2, 2))
conv3 = conv_block(conv3, n_filters * 4)
short3 = conv_block(short3, n_filters * 4, strides=(2, 2), activation=None)
conv3 = Add()([conv3, short3])
# Bridge
short4 = conv3
conv4 = conv_block(conv3, n_filters * 8, strides=(2, 2))
conv4 = conv_block(conv4, n_filters * 8)
short4 = conv_block(short4, n_filters * 8, strides=(2, 2), activation=None)
conv4 = Add()([conv4, short4])
# Expansive path
up5 = Conv2DTranspose(filters=n_filters * 4,
kernel_size=(3, 3),
strides=(2, 2),
padding='same')(conv4)
up5 = concatenate([up5, conv3])
short5 = up5
conv5 = conv_block(up5, n_filters * 4)
conv5 = conv_block(conv5, n_filters * 4)
short5 = conv_block(short5, n_filters * 4, activation=None)
conv5 = Add()([conv5, short5])
up6 = Conv2DTranspose(filters=n_filters * 2,
kernel_size=(3, 3),
strides=(2, 2),
padding='same')(conv5)
up6 = concatenate([up6, conv2])
short6 = up6
conv6 = conv_block(up6, n_filters * 2)
conv6 = conv_block(conv6, n_filters * 2)
short6 = conv_block(short6, n_filters * 2, activation=None)
conv6 = Add()([conv6, short6])
up7 = Conv2DTranspose(filters=n_filters,
kernel_size=(3, 3),
strides=(2, 2),
padding='same')(conv6)
up7 = concatenate([up7, conv1])
short7 = up7
conv7 = conv_block(up7, n_filters)
conv7 = conv_block(conv7, n_filters)
short7 = conv_block(short7, n_filters, activation=None)
conv7 = Add()([conv7, short7])
outputs = Conv2D(filters=1, kernel_size=(1, 1),
activation='sigmoid')(conv7)
model = Model(inputs=[inputs], outputs=[outputs])
return model
def decoder(
x,
output_shape,
encoded_shape,
conv_chans=None,
min_h=5,
min_c=4,
prefix='vte_dec',
n_convs_per_stage=1,
include_dropout=False,
ks=3,
include_skips=None,
use_residuals=False,
use_upsample=False,
use_batchnorm=False,
target_vol_sizes=None,
n_samples=1,
kernel_initializer=None,
bias_initializer=None,
):
n_dims = len(output_shape) - 1
if conv_chans is None:
n_convs = int(np.floor(np.log2(output_shape[0] / min_h)))
conv_chans = [min_c * 2] * n_convs
elif not type(conv_chans) == list:
n_convs = int(np.floor(np.log2(output_shape[0] / min_h)))
conv_chans = [conv_chans] * n_convs
elif type(conv_chans) == list:
n_convs = len(conv_chans)
if isinstance(ks, list):
assert len(ks) == (n_convs + 1
) # specify for each conv, as well as the last one
else:
ks = [ks] * (n_convs + 1)
print('Decoding with conv filters {}'.format(conv_chans))
# compute default sizes that we want on the way up, mainly in case we have more convs than stages
# and we upsample past the output size
if n_dims == 2:
# just upsample by a factor of 2 and then crop the final volume to the desired volume
default_target_vol_sizes = np.asarray(
[(int(encoded_shape[0] * 2.**(i + 1)),
int(encoded_shape[1] * 2.**(i + 1)))
for i in range(n_convs - 1)] + [output_shape[:2]])
else:
print(output_shape)
print(encoded_shape)
# just upsample by a factor of 2 and then crop the final volume to the desired volume
default_target_vol_sizes = np.asarray(
[(min(output_shape[0], int(encoded_shape[0] * 2.**(i + 1))),
min(output_shape[1], int(encoded_shape[1] * 2.**(i + 1))),
min(output_shape[2], int(encoded_shape[2] * 2.**(i + 1))))
for i in range(n_convs - 1)] + [output_shape[:3]])
# automatically stop when we reach the desired image shape
for vi, vs in enumerate(default_target_vol_sizes):
if np.all(vs >= output_shape[:-1]):
default_target_vol_sizes[vi] = output_shape[:-1]
print('Automatically computed target output sizes: {}'.format(
default_target_vol_sizes))
if target_vol_sizes is None:
target_vol_sizes = default_target_vol_sizes
else:
print('Target concat vols to match shapes to: {}'.format(
target_vol_sizes))
# TODO: check that this logic makes sense for more convs
# fill in any Nones that we might have in our target_vol_sizes
filled_target_vol_sizes = [None] * len(target_vol_sizes)
for i in range(n_convs):
if i < len(target_vol_sizes) and target_vol_sizes[i] is not None:
filled_target_vol_sizes[i] = target_vol_sizes[i]
target_vol_sizes = filled_target_vol_sizes
if include_skips is not None:
print('Concatentating padded/cropped shapes {} with skips {}'.format(
target_vol_sizes, include_skips))
curr_shape = np.asarray(encoded_shape[:n_dims])
for i in range(n_convs):
print(target_vol_sizes[i])
if i < len(target_vol_sizes) and target_vol_sizes[i] is not None:
x = _pad_or_crop_to_shape(x, curr_shape, target_vol_sizes[i])
curr_shape = np.asarray(
target_vol_sizes[i]) # we will upsample first thing next stage
# if we want to concatenate with another volume (e.g. from encoder, or a downsampled input)...
if include_skips is not None and i < len(
include_skips) and include_skips[i] is not None:
x_shape = x.get_shape().as_list()
skip_shape = include_skips[i].get_shape().as_list()
print(
'Attempting to concatenate current layer {} with previous skip connection {}'
.format(x_shape, skip_shape))
# input size might not match in time dimension, so just tile it
if n_samples > 1:
tile_factor = [1] + [n_samples] + [1] * (len(x_shape) - 1)
print('Tiling by {}'.format(tile_factor))
print(target_vol_sizes[i])
skip = Lambda(lambda y: K.expand_dims(y, axis=1))(
include_skips[i])
skip = Lambda(lambda y: tf.tile(y, tile_factor),
name='{}_lambdatilesamples_{}'.format(prefix, i),
output_shape=[n_samples] + skip_shape[1:])(skip)
skip = Lambda(lambda y: tf.reshape(y, [-1] + skip_shape[1:]),
output_shape=skip_shape[1:])(skip)
else:
skip = include_skips[i]
x = Concatenate(axis=-1,
name='{}_concatskip_{}'.format(prefix,
i))([x, skip])
for ci in range(n_convs_per_stage):
x = myConv(
conv_chans[i],
ks=ks[i],
strides=1,
n_dims=n_dims,
prefix=prefix,
suffix='{}_{}'.format(i, ci + 1),
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
)(x)
if use_batchnorm: # TODO: check to see if this should go before the residual
x = BatchNormalization()(x)
# if we want residuals, store them here
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_{}_add_residual'.format(prefix, i))(
[residual_input, x])
x = LeakyReLU(0.2,
name='{}_leakyrelu_{}_{}'.format(prefix, i,
ci + 1))(x)
if include_dropout and i < 2:
x = Dropout(0.3)(x)
# if we are not at the output resolution yet, upsample or do a transposed convolution
if not np.all(curr_shape == output_shape[:len(curr_shape)]):
if not use_upsample:
# if we have convolutional filters left at the end, just apply them at full resolution
x = myConvTranspose(
conv_chans[i],
n_dims=n_dims,
ks=ks[i],
strides=2,
prefix=prefix,
suffix=i,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
)(x)
if use_batchnorm:
x = BatchNormalization()(x)
x = LeakyReLU(0.2, name='{}_leakyrelu_{}'.format(prefix, i))(
x) # changed 5/15/2018, will break old models
else:
x = myUpsample(size=2, n_dims=n_dims, prefix=prefix,
suffix=i)(x)
curr_shape *= 2
# last stage of convolutions, no more upsampling
x = myConv(
output_shape[-1],
ks=ks[-1],
n_dims=n_dims,
strides=1,
prefix=prefix,
suffix='final',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
)(x)
return x
def RESNET_BILSTM_model(self, conf, arm_shape):
n_feature_maps = 8 #when you have large dataset increase it to 16 or 32 or 64 , experiment
road_num = arm_shape[0]
A = arm_shape[1]
input_x = Input((road_num, conf.observe_length, 2)) #changed here
input_ram = Input(arm_shape)
input_veh_type = Input((road_num, conf.observe_length, 1))
input_engine = Input((road_num, conf.observe_length, 1))
input_weight = Input((road_num, conf.observe_length, 1))
print('input_x.shape', input_x.shape)
print('input_ram.shape', input_ram.shape)
print('road_num.shape', road_num)
# BLOCK 1
veh_type_embd = Embedding(5, 3, mask_zero=False)(input_veh_type)
engine_embd = Embedding(63, 10, mask_zero=False)(input_engine)
weight_embd = Embedding(10, 5, mask_zero=False)(input_weight)
squeezer = Lambda(lambda x: squeeze(x, axis=-2))
veh_type_embd = squeezer(veh_type_embd)
engine_embd = squeezer(engine_embd)
weight_embd = squeezer(weight_embd)
concat_x = Concatenate()(
[input_x, veh_type_embd, engine_embd, weight_embd])
conv_x = Lookup(conf.batch_size)([concat_x, input_ram])
conv_x = Conv3D(n_feature_maps, (1, A, 2), activation='relu')(conv_x)
conv_x = BatchNormalization()(conv_x)
conv_x = LookUpSqueeze()(conv_x)
conv_y = Lookup(conf.batch_size)([conv_x, input_ram])
conv_y = Conv3D(n_feature_maps, (1, A, 2), activation='relu')(conv_y)
conv_y = BatchNormalization()(conv_y)
conv_y = LookUpSqueeze()(conv_y)
conv_z = Lookup(conf.batch_size)([conv_y, input_ram])
conv_z = Conv3D(n_feature_maps, (1, A, 2))(conv_z)
conv_z = BatchNormalization()(conv_z)
conv_z = LookUpSqueeze()(conv_z)
# expand channels for the sum
shortcut_y = Lookup(conf.batch_size)([concat_x, input_ram])
shortcut_y = Conv3D(n_feature_maps, (1, A, 4),
activation='relu')(shortcut_y)
shortcut_y = BatchNormalization()(shortcut_y)
shortcut_y = LookUpSqueeze()(shortcut_y)
output_block_1 = keras.layers.add([shortcut_y, conv_z])
output_block_1 = Activation('relu')(output_block_1)
# BLOCK 2
conv_x = Lookup(conf.batch_size)([output_block_1, input_ram])
conv_x = Conv3D(n_feature_maps * 2, (1, A, 2),
activation='relu')(conv_x)
conv_x = BatchNormalization()(conv_x)
conv_x = LookUpSqueeze()(conv_x)
conv_y = Lookup(conf.batch_size)([conv_x, input_ram])
conv_y = Conv3D(n_feature_maps * 2, (1, A, 2),
activation='relu')(conv_y)
conv_y = BatchNormalization()(conv_y)
conv_y = LookUpSqueeze()(conv_y)
conv_z = Lookup(conf.batch_size)([conv_y, input_ram])
conv_z = Conv3D(n_feature_maps * 2, (1, A, 2))(conv_z)
conv_z = BatchNormalization()(conv_z)
conv_z = LookUpSqueeze()(conv_z)
# expand channels for the sum
shortcut_y = Lookup(conf.batch_size)([output_block_1, input_ram])
shortcut_y = Conv3D(n_feature_maps * 2, (1, A, 4),
activation='relu')(shortcut_y)
shortcut_y = BatchNormalization()(shortcut_y)
shortcut_y = LookUpSqueeze()(shortcut_y)
print('conv_z.shape', conv_z.shape)
output_block_2 = keras.layers.add([shortcut_y, conv_z])
output_block_2 = Activation('relu')(output_block_2)
# BLOCK 3
conv_x = Lookup(conf.batch_size)([output_block_2, input_ram])
conv_x = Conv3D(n_feature_maps * 2, (1, A, 2),
activation='relu')(conv_x)
conv_x = BatchNormalization()(conv_x)
conv_x = LookUpSqueeze()(conv_x)
conv_y = Lookup(conf.batch_size)([conv_x, input_ram])
conv_y = Conv3D(n_feature_maps * 2, (1, A, 2),
activation='relu')(conv_y)
conv_y = BatchNormalization()(conv_y)
conv_y = LookUpSqueeze()(conv_y)
conv_z = Lookup(conf.batch_size)([conv_y, input_ram])
conv_z = Conv3D(n_feature_maps * 2, (1, A, 2))(conv_z)
conv_z = BatchNormalization()(conv_z)
conv_z = LookUpSqueeze()(conv_z)
# need to expand
shortcut_y = Lookup(conf.batch_size)([output_block_2, input_ram])
shortcut_y = Conv3D(n_feature_maps * 2, (1, A, 4),
activation='relu')(shortcut_y)
shortcut_y = BatchNormalization()(shortcut_y)
shortcut_y = LookUpSqueeze()(shortcut_y)
output_block_3 = keras.layers.add([shortcut_y, conv_z])
output_block_3 = Activation('relu')(output_block_3)
print('output_block_3', output_block_3.shape)
to_lstm = Lookup(conf.batch_size)([output_block_3, input_ram])
print('to_lstm BEFORE EXTERNAL #############', to_lstm.shape)
if conf.use_externel:
to_lstm = Conv3D(n_feature_maps, (1, A, 38),
activation='relu')(to_lstm)
else:
to_lstm = Conv3D(n_feature_maps, (1, A, 1),
activation='relu')(to_lstm)
to_lstm = LookUpSqueeze()(to_lstm)
to_lstm = Lambda(lambda y: squeeze(y, 0))(to_lstm)
# output_block_3 = Activation('relu')(output_block_3)
#FINAL
# gap_layer = GlobalAveragePooling2D()(output_block_3)
# gap_layer = Dropout(rate=.25)(gap_layer)
# gap_layer = Dense(50, activation='sigmoid')(gap_layer)
# gap_layer = Dropout(rate=.25)(gap_layer)
# time_distibuted = TimeDistributed(output_block_3)
print('to_lstm.shape', to_lstm.shape)
# output = SimpleRNN(5)(to_lstm)
# output = MyReshape(conf.batch_size)(gap_layer)
output = Bidirectional(
LSTM(10, return_sequences=True, dropout=0.5,
recurrent_dropout=0.2))(to_lstm)
print('lstm out.shape', output.shape)
#output = MyReshape(conf.batch_size)(output)
inputs = [
input_x, input_ram, input_veh_type, input_engine, input_weight
]
if conf.use_externel:
output = Dense(1, activation='relu')(output)
output = MyInverseReshape2(conf.batch_size)(output)
print('dense.output.shape', output.shape)
input_e, output_e = self.__E_input_output(conf, arm_shape)
print('output_e shape', output_e.shape)
if isinstance(input_e, list):
inputs += input_e
else:
inputs += [input_e]
if conf.use_matrix_fuse:
outputs = [matrixLayer()(output)]
print('outputs', outputs)
print('outputs.e', output_e)
outputs.append(matrixLayer2()(output_e))
print('outputs.shape', outputs)
output = Add()(outputs)
else:
output = Add()([output, output_e])
output = Activation('tanh')(output)
else:
output = Dense(1, activation='tanh')(to_lstm)
output = MyInverseReshape2(conf.batch_size)(output)
output = Dense(conf.predict_length, activation='tanh')(output)
print('final layer', output.shape)
model = Model(inputs=inputs, outputs=output)
return model
def segnet2D(x_in,
img_shape,
out_im_chans,
nf_enc=[64, 64, 128, 128, 256, 256, 512],
nf_dec=None,
regularizer=None,
initializer=None,
layer_prefix='segnet',
n_convs_per_stage=1,
include_residual=False,
concat_at_stages=None):
ks = 3
encoding_im_sizes = np.asarray([(int(np.ceil(img_shape[0]/2.0**i)), int(np.ceil(img_shape[1]/2.0**i))) \
for i in range(0, len(nf_enc) + 1)])
reg_params = {}
if regularizer == 'l1':
reg = regularizers.l1(1e-6)
else:
reg = None
if initializer == 'zeros':
reg_params['kernel_initializer'] = initializers.Zeros()
x = x_in
# start with the input channels
encoder_pool_idxs = []
for i in range(len(nf_enc)):
for j in range(n_convs_per_stage):
x = Conv2D(nf_enc[i],
kernel_regularizer=reg,
kernel_size=ks,
strides=(1, 1),
padding='same',
name='{}_enc_conv2D_{}_{}'.format(
layer_prefix, i, j + 1))(x)
if concat_at_stages and concat_at_stages[i] is not None:
x = Concatenate(axis=-1)([x, concat_at_stages[i]])
if j == 0 and include_residual:
residual_input = x
elif j == n_convs_per_stage - 1 and include_residual:
x = Add()([residual_input, x])
x = LeakyReLU(0.2)(x)
x, pool_idxs = MaxPoolingWithArgmax2D(pool_size=(ks, ks),
padding='same',
name='{}_enc_maxpool_{}'.format(
layer_prefix, i))(x)
encoder_pool_idxs.append(pool_idxs)
if nf_dec is None:
nf_dec = list(reversed(nf_enc[1:]))
decoding_im_sizes = [encoding_im_sizes[-1] * 2]
for i in range(len(nf_dec)):
x = MaxUnpooling2D()([x, encoder_pool_idxs[-1 - i]])
x = _pad_or_crop_to_shape(x, decoding_im_sizes[-1],
encoding_im_sizes[-i - 2])
decoding_im_sizes.append(
encoding_im_sizes[-i - 2] *
2) # the next deconv layer will produce this image height
residual_input = x
for j in range(n_convs_per_stage):
x = Conv2D(nf_dec[i],
kernel_regularizer=reg,
kernel_size=ks,
strides=(1, 1),
padding='same',
name='{}_dec_conv2D_{}_{}'.format(layer_prefix, i,
j))(x)
if j == 0 and include_residual:
residual_input = x
elif j == n_convs_per_stage - 1 and include_residual:
x = Add()([residual_input, x])
x = LeakyReLU(0.2)(x)
if i < len(nf_dec) - 1:
# an extra conv compared to unet, so that the unpool op gets the right number of filters
x = Conv2D(nf_dec[i + 1],
kernel_regularizer=reg,
kernel_size=ks,
strides=(1, 1),
padding='same',
name='{}_dec_conv2D_{}_extra'.format(layer_prefix,
i))(x)
x = LeakyReLU(0.2)(x)
y = Conv2D(out_im_chans,
kernel_size=1,
padding='same',
kernel_regularizer=reg,
name='{}_dec_conv2D_last_last'.format(layer_prefix))(
x) # add your own activation after this model
# add your own activation after this model
return y
def NIC(max_token_length,
vocabulary_size,
embedding_matrix,
rnn='lstm',
num_image_features=2048,
hidden_size=512,
embedding_size=512,
regularizer=1e-8):
embedding_layer = Embedding(vocabulary_size,
50,
weights=[embedding_matrix],
input_length=max_token_length,
trainable=False)
# word embedding
text_input = Input(shape=(max_token_length, ), name='text')
text_dropout = embedding_layer(text_input)
#text_mask = Masking(mask_value=0.0, name='text_mask')(text_input)
#text_to_embedding = TimeDistributed(Dense(units=embedding_size,
# kernel_regularizer=l2(regularizer),
# name='text_embedding'))(text_mask)
#text_dropout = Dropout(.5, name='text_dropout')(text_to_embedding)
# image embedding
image_input = Input(shape=(max_token_length, num_image_features),
name='image')
image_embedding = TimeDistributed(
Dense(units=embedding_size,
kernel_regularizer=l2(regularizer),
name='image_embedding'))(image_input)
image_dropout = Dropout(.5, name='image_dropout')(image_embedding)
# language model
recurrent_inputs = [text_dropout, image_dropout]
merged_input = Add()(recurrent_inputs)
if rnn == 'lstm':
recurrent_network = LSTM(units=hidden_size,
recurrent_regularizer=l2(regularizer),
kernel_regularizer=l2(regularizer),
bias_regularizer=l2(regularizer),
return_sequences=True,
name='recurrent_network')(merged_input)
elif rnn == 'gru':
recurrent_network = GRU(units=hidden_size,
recurrent_regularizer=l2(regularizer),
kernel_regularizer=l2(regularizer),
bias_regularizer=l2(regularizer),
return_sequences=True,
name='recurrent_network')(merged_input)
else:
raise Exception('Invalid rnn name')
output = TimeDistributed(Dense(units=vocabulary_size,
kernel_regularizer=l2(regularizer),
activation='softmax'),
name='output')(recurrent_network)
inputs = [text_input, image_input]
model = Model(inputs=inputs, outputs=output)
return model
def model(nbChannels,
fs,
stride=2,
embeddingRate=30.0,
nbEmbeddingDim=32,
nbQuantizationBins=16,
nbResidualLayers=2,
**kwargs):
inputs = Input(shape=(None, nbChannels))
x = inputs
nbFilters = 64
kernelSize = 3
with K.name_scope('encoder'):
i = 0
fs = fs
while fs > embeddingRate:
fs = int(np.ceil(fs / stride))
# Projection shortcut using resampling
xs = Resample1D(scale=1.0 / stride)(x)
# Residual blocks
for _ in range(nbResidualLayers):
fx = Conv1D(nbFilters,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same')(x)
fx = PReLU(alpha_initializer=Constant(0.25),
shared_axes=[1])(fx)
fx = Conv1D(nbFilters,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same')(fx)
x = Add()([fx, x])
x = PReLU(alpha_initializer=Constant(0.25),
shared_axes=[1])(x)
# Non-linear convolution followed by linear downsampling
x = Conv1D(nbFilters,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-ds-%d' % (i + 1))(x)
x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x)
x = Resample1D(scale=1.0 / stride, method='linear')(x)
x = Add()([x, xs])
i += 1
nbDownsamplingLayers = i
x = Conv1D(nbEmbeddingDim,
kernel_size=3,
strides=1,
activation='linear',
padding='same',
name='codec-conv-embedding')(x)
x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x)
# Quantization
if nbQuantizationBins is not None:
quantization = SoftmaxQuantization(nbQuantizationBins,
name='quantization')
x = quantization(x)
# Dequantization
if nbQuantizationBins is not None:
x = SoftmaxDequantization(nbQuantizationBins)(x)
with K.name_scope('decoder'):
x = Conv1D(nbFilters,
kernel_size=1,
strides=1,
activation='linear',
padding='same')(x)
x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x)
# Upsampling with resize convolutions
for i in range(nbDownsamplingLayers):
# Projection shortcut
xs = Resample1D(stride, method='linear')(x)
# Linear upsampling followed by non-linear convolution
x = Resample1D(stride, method='linear')(x)
x = Conv1D(nbFilters,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-us-%d' % (i + 1))(x)
x = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(x)
# Residual blocks
for _ in range(nbResidualLayers):
fx = Conv1D(nbFilters,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same')(x)
fx = PReLU(alpha_initializer=Constant(0.25),
shared_axes=[1])(fx)
fx = Conv1D(nbFilters,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same')(fx)
x = Add()([fx, x])
x = PReLU(alpha_initializer=Constant(0.25),
shared_axes=[1])(x)
x = Add()([x, xs])
xr = Conv1D(nbChannels,
kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-out')(x)
outputs = xr
return Model(inputs, outputs, **kwargs)
def add_resnet(x, n_dim):
in_x = x
x = Dense(n_dim, activation="relu")(x)
x = Dense(n_dim, activation="relu")(x)
return Add()([in_x, x])
def modelv3(nbChannels,
stride=2,
nbFilters=[64, 64, 128, 128, 256],
nbEmbeddingDim=32,
nbQuantizationBins=16,
nbResidualLayers=1,
**kwargs):
inputs = Input(shape=(None, nbChannels))
x = inputs
def _gated_block(x, kernelSize, nbFilters, noisy=False, name=None):
# Define the gated activation unit for the input
f = Conv1D(nbFilters,
kernelSize,
padding='same',
activation='linear',
name=name)(x) # filter
f = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(f)
g = Conv1D(nbFilters,
kernelSize,
padding='same',
activation='sigmoid')(x) # gate
zi = Multiply()([f, g])
if noisy:
# Define the gated activation unit for the noise
n = Lambda(lambda x: K.random_normal(
shape=K.shape(x), mean=0.0, stddev=1.0))(x)
f = Conv1D(nbFilters,
kernelSize,
padding='same',
activation='linear')(n) # filter
f = PReLU(alpha_initializer=Constant(0.25), shared_axes=[1])(f)
g = Conv1D(nbFilters,
kernelSize,
padding='same',
activation='sigmoid')(x) # gate
zn = Multiply()([f, g])
# Add signal and noise
y = Add()([zi, zn])
else:
y = zi
# Define the parametrized skip connection
skip = Conv1D(nbFilters, 1, padding='same')(y)
return y, skip
kernelSize = 5
with K.name_scope('encoder'):
for i, n in enumerate(nbFilters):
# Non-linear convolution followed by linear downsampling
skips = []
for _ in range(nbResidualLayers):
x, skip = _gated_block(x,
kernelSize,
n,
noisy=False,
name='codec-conv-ds-%d' % (i + 1))
skips.append(skip)
if len(skips) > 1:
x = Add()(skips)
else:
x = skips[0]
x = Resample1D(scale=1.0 / stride, method='linear')(x)
x = Conv1D(nbEmbeddingDim,
kernel_size=kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-embedding')(x)
# Quantization
if nbQuantizationBins is not None:
quantization = SoftmaxQuantization(nbQuantizationBins,
name='quantization')
x = quantization(x)
# Dequantization
if nbQuantizationBins is not None:
x = SoftmaxDequantization(nbQuantizationBins)(x)
kernelSize = 5
with K.name_scope('decoder'):
# Upsampling with resize convolutions
for i, n in enumerate(reversed(nbFilters)):
# Linear upsampling followed by non-linear convolution
x = Resample1D(stride, method='linear')(x)
skips = []
for _ in range(nbResidualLayers):
x, skip = _gated_block(x,
kernelSize,
n,
noisy=False,
name='codec-conv-us-%d' % (i + 1))
skips.append(skip)
if len(skips) > 1:
x = Add()(skips)
else:
x = skips[0]
xr = Conv1D(nbChannels,
kernelSize,
strides=1,
activation='linear',
padding='same',
name='codec-conv-out')(x)
outputs = xr
return Model(inputs, outputs, **kwargs)
def add_conv(input_dim, tobj, l):
return Add()([
conv131(input_dim, l, tobj=get(tobj, 1), first_layer=True),
Conv(input_dim * 2, 1, tobj=get(tobj, 12))(l)
])
def ThreeLevelsMaxPool(
input_dims,
num_classes,
init='glorot_normal',
encoder_act_function='elu',
decoder_act_function='relu',
classification_act_function='softmax',
loss_function='categorical_crossentropy',
learning_rate=1e-05,
min_filters_per_layer=16,
use_kernel_reg=False,
use_dropout=False,
):
if use_kernel_reg == True:
lvl2_reg = l2(0.005)
lvl3_reg = l2(0.01)
else:
lvl2_reg = None
lvl3_reg = None
if use_dropout == True:
lvl1_dropout = 0.0
lvl2_dropout = 0.25
lvl3_dropout = 0.5
else:
lvl1_dropout = 0.0
lvl2_dropout = 0.0
lvl3_dropout = 0.0
lvl1_filters = (3, 3, 3)
lvl2_filters = (3, 3, 3)
lvl3_filters = (3, 3, 3)
# -----------------------------------
# ENCODER - LEVEL 1
# -----------------------------------
with tf.device('/gpu:0'):
inputs = Input(input_dims, name='main_input')
enc_lvl1_block1 = myConv3DBlock(
input_tensor=inputs,
filters=min_filters_per_layer,
kernel_size=lvl1_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='enc_lvl1a',
kernel_regularizer=None,
use_batch_norm=False,
dropout_rate=lvl1_dropout,
)
max_pool_1to2 = MaxPooling3D(pool_size=(4, 4, 4),
name='max_pool_1to2')(enc_lvl1_block1)
# -----------------------------------
# ENCODER - LEVEL 2
# -----------------------------------
enc_lvl2_block1 = myConv3DBlock(
input_tensor=max_pool_1to2,
filters=2 * min_filters_per_layer,
kernel_size=lvl2_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='enc_lvl2a',
kernel_regularizer=lvl2_reg,
use_batch_norm=False,
dropout_rate=lvl2_dropout,
)
with tf.device('/gpu:1'):
enc_lvl2_block2 = myConv3DBlock(
input_tensor=enc_lvl2_block1,
filters=2 * min_filters_per_layer,
kernel_size=lvl2_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='enc_lvl2b',
kernel_regularizer=lvl2_reg,
use_batch_norm=False,
dropout_rate=lvl2_dropout,
)
max_pool_2to3 = MaxPooling3D(pool_size=(2, 2, 2),
name='max_pool_2to3')(enc_lvl2_block2)
# -----------------------------------
# BOTTLENECK LAYER
# -----------------------------------
bottleneck_block1 = myConv3DBlock(
input_tensor=max_pool_2to3,
filters=4 * min_filters_per_layer,
kernel_size=lvl3_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='bottleneck_a',
kernel_regularizer=lvl3_reg,
use_batch_norm=False,
dropout_rate=lvl3_dropout,
)
bottleneck_block2 = myConv3DBlock(
input_tensor=bottleneck_block1,
filters=4 * min_filters_per_layer,
kernel_size=lvl3_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='bottleneck_b',
kernel_regularizer=lvl3_reg,
use_batch_norm=False,
dropout_rate=lvl3_dropout,
)
bottleneck_block3 = myConv3DBlock(
input_tensor=bottleneck_block2,
filters=4 * min_filters_per_layer,
kernel_size=lvl3_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='bottleneck_c',
kernel_regularizer=lvl3_reg,
use_batch_norm=False,
dropout_rate=lvl3_dropout,
)
# -----------------------------------
# DECODER - LEVEL 2
# -----------------------------------
up_conv_3to2 = myUpConv3DBlock(
input_tensor=bottleneck_block3,
filters=2 * min_filters_per_layer,
kernel_size=(2, 2, 2),
strides=(2, 2, 2),
kernel_initializer=init,
padding='same',
block_name='up_conv_3to2',
kernel_regularizer=lvl2_reg,
use_batch_norm=False,
dropout_rate=lvl2_dropout,
)
with tf.device('/gpu:2'):
skip_conn_lvl2 = Add(name='lvl2_longskip')(
[enc_lvl2_block2, up_conv_3to2])
dec_lvl2_block1 = myConv3DBlock(
input_tensor=skip_conn_lvl2,
filters=2 * min_filters_per_layer,
kernel_size=lvl2_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='dec_lvl2a',
kernel_regularizer=lvl2_reg,
use_batch_norm=False,
dropout_rate=lvl2_dropout,
)
dec_lvl2_block2 = myConv3DBlock(
input_tensor=dec_lvl2_block1,
filters=2 * min_filters_per_layer,
kernel_size=lvl2_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='dec_lvl2b',
kernel_regularizer=lvl2_reg,
use_batch_norm=False,
dropout_rate=lvl2_dropout,
)
# -----------------------------------
# DECODER - LEVEL 1
# -----------------------------------
up_conv_2to1 = myUpConv3DBlock(
input_tensor=dec_lvl2_block2,
filters=min_filters_per_layer,
kernel_size=(4, 4, 4),
strides=(4, 4, 4),
kernel_initializer=init,
padding='same',
block_name='up_conv_2to1',
kernel_regularizer=None,
use_batch_norm=False,
dropout_rate=lvl2_dropout,
)
skip_conn_lvl1 = Add(name='lvl1_longskip')(
[enc_lvl1_block1, up_conv_2to1])
with tf.device('/gpu:3'):
dec_lvl1_block1 = myConv3DBlock(
input_tensor=skip_conn_lvl1,
filters=min_filters_per_layer,
kernel_size=lvl1_filters,
activation=encoder_act_function,
kernel_initializer=init,
padding='same',
block_name='dec_lvl1a',
kernel_regularizer=None,
use_batch_norm=False,
dropout_rate=lvl1_dropout,
)
outputs = Conv3D(
filters=num_classes,
kernel_size=(1, 1, 1),
activation=classification_act_function,
kernel_initializer=init,
name='main_output',
)(dec_lvl1_block1)
# define the model object and the optimizer
model = Model(inputs=[inputs], outputs=[outputs])
my_adam = keras.optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999)
# compile the model
model.compile(loss=losses_dict[loss_function],
optimizer=my_adam,
metrics=[
'categorical_crossentropy',
losses_dict['multiclass_dice_coeff'],
losses_dict['average_multiclass_dice_coeff'],
losses_dict['tanimoto_coefficient']
])
# return the model object
return model
def add_id(input_dim, tobj, l):
return Add()(
[conv131(input_dim, l, tobj=get(tobj, 1), first_layer=False), l])
# 모델 2 input2 = Input(shape=(3, 1)) model2 = LSTM(6, activation='relu')(input1) dense11 = Dense(4)(model2) dense21 = Dense(3)(dense11) dense31 = Dense(2)(dense21) model2 = Dense(1)(dense31) # 1번째 모델 합치는 방법 #from keras.layers.merge import concatenate #merge1 = concatenate([output1,output2]) # input 모델 합치기 # concatenate는 모델 1,2의 아웃풋 노드수가 달라도 된다. 엮는것이므로 # 2번째 방법 from keras.layers.merge import Add merge1 = Add()([model1, model2]) # 모델1,2 의 아웃풋 노드수가 같아야 한다. middle1 = Dense(4)(merge1) middle2 = Dense(3)(middle1) # 1번째 output output_1 = Dense(2)(middle2) output_1 = Dense(1)(output_1) # y의 벡터가 1개이므로 1로 설정 # 2번째 output output_2 = Dense(3)(middle2) output_2 = Dense(1)(output_2) model = Model(inputs=[input1, input2], outputs=[output_1, output_2]) model.summary()
def encoder3D(x, img_shape,
conv_chans=None,
n_convs_per_stage=1,
min_h=5, min_c=None,
prefix='vte',
ks=3,
return_skips=False, use_residuals=False, use_maxpool=False,
max_time_downsample=None):
skip_layers = []
concat_skip_sizes = []
if max_time_downsample is None:
# do not attempt to downsample beyond 1
max_time_downsample = int(np.floor(np.log2(img_shape[-2]))) - 1
print('Max downsamples in time: {}'.format(max_time_downsample))
if conv_chans is None:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [min_c * 2] * (n_convs - 1) + [min_c]
elif not type(conv_chans) == list:
n_convs = int(np.floor(np.log2(img_shape[0] / min_h)))
conv_chans = [conv_chans] * (n_convs - 1) + [min_c]
for i in range(len(conv_chans)):
if n_convs_per_stage is not None and n_convs_per_stage > 1 or use_maxpool and n_convs_per_stage is not None:
for ci in range(n_convs_per_stage):
x = Conv3D(conv_chans[i], kernel_size=ks, padding='same',
name='{}_enc_conv3D_{}_{}'.format(prefix, i, ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_enc_{}_add_residual'.format(prefix, i))([residual_input, x])
x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}_{}'.format(prefix, i, ci + 1))(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(x.get_shape().as_list()[1:-1])
# only downsample if we are below the max number of downsamples in time
if i < max_time_downsample:
strides = (2, 2, 2)
else:
strides = (2, 2, 1)
if use_maxpool:
x = MaxPooling3D(pool_size=strides,
name='{}_enc_maxpool_{}'.format(prefix, i))(x)
else:
x = Conv3D(conv_chans[i], kernel_size=ks, strides=strides, padding='same',
name='{}_enc_conv3D_{}'.format(prefix, i))(x)
if i < len(conv_chans) - 1: # no activation on last convolution
x = LeakyReLU(0.2, name='{}_enc_leakyrelu_{}'.format(prefix, i))(x)
if min_c is not None and min_c > 0:
if n_convs_per_stage is not None and n_convs_per_stage > 1:
for ci in range(n_convs_per_stage):
x = Conv3D(min_c, kernel_size=ks, padding='same',
name='{}_enc_conv3D_last_{}'.format(prefix, ci + 1))(x)
if ci == 0 and use_residuals:
residual_input = x
elif ci == n_convs_per_stage - 1 and use_residuals:
x = Add(name='{}_enc_{}_add_residual'.format(prefix, 'last'))([residual_input, x])
x = LeakyReLU(0.2, name='{}_enc_leakyrelu_last'.format(prefix))(x)
x = Conv3D(min_c, kernel_size=ks, strides=(1, 1, 1), padding='same',
name='{}_enc_conv3D_last'.format(prefix))(x)
if return_skips:
skip_layers.append(x)
concat_skip_sizes.append(x.get_shape().as_list()[1:-1])
if return_skips:
return x, skip_layers, concat_skip_sizes
else:
return x
