def model(input_shape, input_shape2):
up_0 = Input(shape=input_shape, name='up_stream_0')
up_1 = Input(shape=input_shape, name='up_stream_1')
down_0 = Input(shape=input_shape, name='down_stream_0')
down_1 = Input(shape=input_shape, name='down_stream_1')
down_02 = Input(shape=input_shape, name='down_stream_02')
down_12 = Input(shape=input_shape, name='down_stream_12')
down_2 = Input(shape=input_shape2, name='down_stream_2')
down_3 = Input(shape=input_shape2, name='down_stream_3')
up_stream = share_stream(x_shape=input_shape, outputNum=128)
down_stream = share_stream(x_shape=input_shape, outputNum=128)
down_stream3 = share_stream(x_shape=input_shape2, outputNum=128)
up_feature_0 = up_stream(up_0)
up_feature_1 = up_stream(up_1)
down_feature_0 = down_stream(down_0)
down_feature_1 = down_stream(down_1)
down_feature_02 = down_stream(down_02)
down_feature_12 = down_stream(down_12)
down_feature_2 = down_stream3(down_2)
down_feature_3 = down_stream3(down_3)
up_feature = Maximum()([up_feature_0, up_feature_1])
down_feature = Maximum()([down_feature_0, down_feature_1])
down_feature2 = Maximum()([down_feature_02, down_feature_12])
down_feature3 = Maximum()([down_feature_2, down_feature_3])
feature = concatenate(
[up_feature, down_feature, down_feature2, down_feature3])
x = Dense(units=256, use_bias=True, trainable=Trainable)(feature)
x2 = Dense(16, trainable=Trainable)(Lambda(lambda x: x * 10000000.)(
Activation("tanh")(Lambda(lambda x: x / 10000000.)(x))))
x2Shape2 = x2._keras_shape
temp = Multiply()([
Dense(256, trainable=Trainable)(Lambda(
lambda x: x * 10000000., output_shape=x2Shape2[1:])(
Activation("tanh")(Lambda(lambda x: x / 10000000.,
output_shape=x2Shape2[1:])(x2)))), x
])
for jj in range(2, taylor):
temp = Add()([
Multiply()([
Dense(256, trainable=Trainable)(
Lambda(lambda x: x * 10000000.,
output_shape=x2Shape2[1:])(Activation("tanh")(
Lambda(lambda x: x / 10000000.,
output_shape=x2Shape2[1:])(x2)))),
Lambda(lambda x: (x**jj) / math.factorial(jj))(x)
]), temp
])
temp = Add()([
Dense(256, trainable=Trainable)(Lambda(
lambda x: x * 10000000., output_shape=x2Shape2[1:])(
Activation("tanh")(Lambda(lambda x: x / 10000000.,
output_shape=x2Shape2[1:])(x2)))),
temp
])
fc_1 = Lambda(lambda x: x * 10000000.)(Activation("tanh")(
Lambda(lambda x: x / 10000000.)(temp)))
fc_1 = Dropout(0.4)(fc_1)
fc_1 = Dense(256, trainable=True)(fc_1)
feature = Dense(units=256, use_bias=True, trainable=True)(feature)
feature = add([feature, fc_1])
feature = Dropout(0.8)(feature)
fc_2 = Dense(units=128, activation='relu', use_bias=True,
trainable=True)(feature)
fc_2 = Dropout(0.4)(fc_2)
fc_2 = Dense(units=128, trainable=Trainable)(fc_2)
feature = Dense(units=128, use_bias=True, trainable=True)(feature)
feature = add([feature, fc_2])
feature = Dropout(0.3)(feature)
fc_4 = Dense(
units=55,
use_bias=True,
)(feature)
fc_4 = Activation('softmax')(fc_4)
network = Model(
input=[up_0, up_1, down_0, down_1, down_02, down_12, down_2, down_3],
outputs=fc_4)
return network
# state transition function
messages = Dense(atom_features, activation='relu')(messages)
messages = Dense(atom_features)(messages)
atom_state = Add()([original_atom_state, messages])
return atom_state, bond_state
for i in range(num_messages):
atom_state, bond_state = message_block(atom_state, bond_state,
connectivity, i)
bond_state = Dense(1)(bond_state)
bond_state = Add()([bond_state, bond_mean])
symb_inputs = [
mol_type, node_graph_indices, bond_graph_indices, atom_types, bond_types,
connectivity
]
model = GraphModel(symb_inputs, [bond_state])
epochs = 500
model.compile(optimizer=keras.optimizers.Adam(lr=lr, decay=decay),
loss=masked_mean_absolute_error)
if not os.path.exists(model_name):
os.makedirs(model_name)
# state transition function
messages = Dense(atom_features, activation='softplus')(messages)
messages = Dense(atom_features)(messages)
atom_state = Add()([atom_state, messages])
return atom_state, bond_state
for _ in range(3):
atom_state, bond_state = message_block(atom_state, bond_state,
connectivity)
atom_state = Dense(atom_features // 2, activation='softplus')(atom_state)
atom_state = Dense(1)(atom_state)
atom_state = Add()([atom_state, atomwise_energy])
output = ReduceAtomToMol(reducer='mean')([atom_state, snode_graph_indices])
model = GraphModel(
[node_graph_indices, atom_types, distance_rbf, connectivity], [output])
lr = 1E-4
epochs = 500
model.compile(optimizer=keras.optimizers.Adam(lr=lr, decay=1E-5), loss='mae')
model.summary()
if not os.path.exists(model_name):
os.makedirs(model_name)
def build_model(self):
### Build VGG-16 ###
content_input = Input(shape=self.batch_shape)
# Block 1
conv1_1 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1')(content_input)
conv1_2 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2')(conv1_1)
pool1_1 = MaxPooling2D((2, 2), strides=(2, 2),
name='block1_pool')(conv1_2)
# Block 2
conv2_1 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1')(pool1_1)
conv2_2 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2')(conv2_1)
pool2_1 = MaxPooling2D((2, 2), strides=(2, 2),
name='block2_pool')(conv2_2)
# Block 3
conv3_1 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1')(pool2_1)
conv3_2 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2')(conv3_1)
conv3_3 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3')(conv3_2)
pool3_1 = MaxPooling2D((2, 2), strides=(2, 2),
name='block3_pool')(conv3_3)
# Block 4
conv4_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1')(pool3_1)
conv4_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2')(conv4_1)
conv4_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3')(conv4_2)
pool4_1 = MaxPooling2D((2, 2), strides=(2, 2),
name='block4_pool')(conv4_3)
# Block 5
conv5_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1')(pool4_1)
conv5_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2')(conv5_1)
conv5_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3')(conv5_2)
pool5_1 = MaxPooling2D((2, 2), strides=(2, 2),
name='block5_pool')(conv5_3)
self.vgg = Model(input=content_input, output=pool5_1)
### Build FCN-8s ###
# Fully-Connected layers
conv6 = Conv2D(4096, (7, 7),
activation='relu',
padding='same',
name='fcn_fc1')(pool5_1)
drop6 = Dropout(0.5)(conv6)
conv7 = Conv2D(4096, (1, 1),
activation='relu',
padding='same',
name='fcn_fc2')(drop6)
drop7 = Dropout(0.5)(conv7)
# Classifier
# conv8 = Conv2D(7, (1, 1), strides=(1, 1), kernel_initializer='he_normal', activation='linear', padding='valid')(drop7)
# up8 = Conv2DTranspose(7, (64, 64), strides=(32, 32), padding='same', name='deconv')(conv8)
# up8 = Conv2DTranspose(7, (32, 32), strides=(16, 16), padding='same', use_bias=False, name='deconv9_1')(conv8)
# pad9 = ZeroPadding2D(padding=(1, 1), name='padding9')(up8)
# up10 = Conv2DTranspose(7, (4, 4), strides=(2, 2), padding='same', use_bias=False, name='deconv9_2')(pad9)
# output = Activation('softmax')(up10)
conv8 = Conv2D(7, (1, 1),
strides=(1, 1),
kernel_initializer='he_normal',
activation='linear',
padding='valid',
name='conv8')(drop7)
up8 = Conv2DTranspose(7, (4, 4),
strides=(2, 2),
padding='same',
use_bias=False,
name='deconv8_1')(conv8)
scaled_pool4 = Lambda(lambda x: x * 1,
name='scaled_pool4')(pool4_1) # 0.01 before
conv9 = Conv2D(7, (1, 1),
strides=(1, 1),
kernel_initializer='he_normal',
padding='valid',
name='conv9')(scaled_pool4)
o1, o2 = self.crop(up8, conv9, content_input)
add9 = Add()([o1, o2])
up9 = Conv2DTranspose(7, (4, 4),
strides=(2, 2),
padding='same',
use_bias=False,
name='deconv9_1')(add9)
scaled_pool3 = Lambda(lambda x: x * 0.1,
name='scaled_pool3')(pool3_1) # 0.0001 before
conv10 = Conv2D(7, (1, 1),
strides=(1, 1),
kernel_initializer='he_normal',
padding='valid',
name='conv10')(scaled_pool3)
o1, o2 = self.crop(up9, conv10, content_input)
add10 = Add()([o1, o2])
up11 = Conv2DTranspose(7, (16, 16),
strides=(8, 8),
padding='same',
use_bias=False,
name='deconv10_1')(add10)
output = Activation('softmax')(up11)
self.model = Model(input=content_input, output=output)
if self.mode == 'train':
print('Loading VGG16 weights...')
self.vgg.load_weights(self.vgg_path, by_name=True)
for layer in self.vgg.layers:
layer.trainable = False
print('VGG16 weights loaded!')
elif self.mode == 'eval':
print('Loading model from {}...'.format(self.model_path))
self.model.load_weights(self.model_path, by_name=True)
print('Model weights loaded!')
return self.model
def get_decoder(self, inputs, name='decoder'):
"""Builds the decoder of a LinkNet architecture.
Args:
name (string, optional): The encoder model name.
Default: 'decoder'.
Returns:
The decoder as a Keras model instance.
"""
# Decoder inputs
encoder4 = Input(shape=int_shape(inputs[0])[1:], name='encoder4')
encoder3 = Input(shape=int_shape(inputs[1])[1:], name='encoder3')
encoder2 = Input(shape=int_shape(inputs[2])[1:], name='encoder2')
encoder1 = Input(shape=int_shape(inputs[3])[1:], name='encoder1')
initial2 = Input(shape=int_shape(inputs[4])[1:], name='initial2')
initial1 = inputs[5]
# Decoder blocks
decoder4 = self.decoder_block(
encoder4,
self.initial_block_filters * 4,
strides=2,
output_shape=int_shape(encoder3)[1:],
bias=self.bias,
name=name + '/4'
)
decoder4 = Add(name=name + '/shortcut_e3_d4')([encoder3, decoder4])
decoder3 = self.decoder_block(
decoder4,
self.initial_block_filters * 2,
strides=2,
output_shape=int_shape(encoder2)[1:],
bias=self.bias,
name=name + '/3'
)
decoder3 = Add(name=name + '/shortcut_e2_d3')([encoder2, decoder3])
decoder2 = self.decoder_block(
decoder3,
self.initial_block_filters,
strides=2,
output_shape=int_shape(encoder1)[1:],
bias=self.bias,
name=name + '/2'
)
decoder2 = Add(name=name + '/shortcut_e1_d2')([encoder1, decoder2])
decoder1 = self.decoder_block(
decoder2,
self.initial_block_filters,
strides=1,
output_shape=int_shape(initial2)[1:],
bias=self.bias,
name=name + '/1'
)
decoder1 = Add(name=name + '/shortcut_init_d1')([initial2, decoder1])
# Final block
# Build the output shape of the next layer - same width and height
# as initial1
shape = (
int_shape(initial1)[1],
int_shape(initial1)[2],
self.initial_block_filters // 2,
)
final = Conv2DTranspose(
self.initial_block_filters // 2,
kernel_size=3,
strides=2,
padding='same',
output_shape=shape,
use_bias=self.bias,
name=name + '/0/transposed2d_1'
)(decoder1)
final = BatchNormalization(name=name + '/0/bn_1')(final)
final = Activation('relu', name=name + '/0/relu_1')(final)
final = Conv2D(
self.initial_block_filters // 2,
kernel_size=3,
padding='same',
use_bias=self.bias,
name=name + '/0/conv2d_1'
)(final)
final = BatchNormalization(name=name + '/0/bn_2')(final)
final = Activation('relu', name=name + '/0/relu_2')(final)
logits = Conv2DTranspose(
self.num_classes,
kernel_size=2,
strides=2,
padding='same',
output_shape=self.output_shape,
use_bias=self.bias,
name=name + '/0/transposed2d_2'
)(final)
prediction = Softmax(name=name + '/0/softmax')(logits)
return Model(
inputs=[
encoder4, encoder3, encoder2, encoder1, initial2
],
outputs=prediction,
name=name
)
def get_effnet_model(save_path,
model_res=1024,
image_size=256,
depth=1,
size=3,
activation='elu',
loss='logcosh',
optimizer='adam'):
if os.path.exists(save_path):
print('Loading model')
return load_model(save_path)
# Build model
print('Building model')
model_scale = int(2 *
(math.log(model_res, 2) - 1)) # For example, 1024 -> 18
if (size <= 0):
effnet = EfficientNetB0(include_top=False,
weights='imagenet',
input_shape=(image_size, image_size, 3))
if (size == 1):
effnet = EfficientNetB1(include_top=False,
weights='imagenet',
input_shape=(image_size, image_size, 3))
if (size == 2):
effnet = EfficientNetB2(include_top=False,
weights='imagenet',
input_shape=(image_size, image_size, 3))
if (size >= 3):
effnet = EfficientNetB3(include_top=False,
weights='imagenet',
input_shape=(image_size, image_size, 3))
layer_size = model_scale * 8 * 8 * 8
if is_square(layer_size): # work out layer dimensions
layer_l = int(math.sqrt(layer_size) + 0.5)
layer_r = layer_l
else:
layer_m = math.log(math.sqrt(layer_size), 2)
layer_l = 2**math.ceil(layer_m)
layer_r = layer_size // layer_l
layer_l = int(layer_l)
layer_r = int(layer_r)
x_init = None
inp = Input(shape=(image_size, image_size, 3))
x = effnet(inp)
if (size < 1):
x = Conv2D(model_scale * 8, 1, activation=activation)(x) # scale down
if (depth > 0):
x = Reshape((layer_r, layer_l))(
x
) # See https://github.com/OliverRichter/TreeConnect/blob/master/cifar.py - TreeConnect inspired layers instead of dense layers.
else:
if (depth < 1):
depth = 1
if (size <= 2):
x = Conv2D(model_scale * 8 * 4, 1,
activation=activation)(x) # scale down a bit
x = Reshape((layer_r * 2, layer_l * 2))(
x
) # See https://github.com/OliverRichter/TreeConnect/blob/master/cifar.py - TreeConnect inspired layers instead of dense layers.
else:
x = Reshape((384, 256))(x) # full size for B3
while (depth > 0):
x = LocallyConnected1D(layer_r, 1, activation=activation)(x)
x = Permute((2, 1))(x)
x = LocallyConnected1D(layer_l, 1, activation=activation)(x)
x = Permute((2, 1))(x)
if x_init is not None:
x = Add()([x, x_init]) # add skip connection
x_init = x
depth -= 1
if (
size >= 2
): # add unshared layers at end for different sections of the latent space
x_init = x
if layer_r % 3 == 0 and layer_l % 3 == 0:
a = LocallyConnected1D(layer_r, 1, activation=activation)(x)
b = LocallyConnected1D(layer_r, 1, activation=activation)(x)
c = LocallyConnected1D(layer_r, 1, activation=activation)(x)
a = Permute((2, 1))(a)
b = Permute((2, 1))(b)
c = Permute((2, 1))(c)
a = LocallyConnected1D(layer_l // 3, 1, activation=activation)(a)
b = LocallyConnected1D(layer_l // 3, 1, activation=activation)(b)
c = LocallyConnected1D(layer_l // 3, 1, activation=activation)(c)
x = Concatenate()([a, b, c])
else:
a = LocallyConnected1D(layer_l, 1, activation=activation)(x)
b = LocallyConnected1D(layer_l, 1, activation=activation)(x)
a = Permute((2, 1))(a)
b = Permute((2, 1))(b)
a = LocallyConnected1D(layer_r // 2, 1, activation=activation)(a)
b = LocallyConnected1D(layer_r // 2, 1, activation=activation)(b)
x = Concatenate()([a, b])
x = Add()([x, x_init]) # add skip connection
x = Reshape((model_scale, 512))(x) # train against all dlatent values
model = Model(inputs=inp, outputs=x)
model.compile(loss=loss, metrics=[], optimizer=optimizer
) # By default: adam optimizer, logcosh used for loss.
return model
def get_resnet_model(save_path, model_res=1024, image_size=256, depth=2, size=0, activation='elu', loss='logcosh',
optimizer='adam'):
# Build model
if os.path.exists(save_path):
print('Loading model')
return load_model(save_path)
print('Building model')
model_scale = int(2*(math.log(model_res, 2)-1)) # For example, 1024 -> 18
if size <= 0:
from keras.applications.resnet50 import ResNet50
resnet = ResNet50(include_top=False, pooling=None, weights='imagenet', input_shape=(image_size, image_size, 3))
else:
from keras_applications.resnet_v2 import ResNet50V2, ResNet101V2, ResNet152V2
if size == 1:
resnet = ResNet50V2(include_top=False, pooling=None, weights='imagenet',
input_shape=(image_size, image_size, 3), backend=keras.backend, layers=keras.layers,
models=keras.models, utils=keras.utils)
if size == 2:
resnet = ResNet101V2(include_top=False, pooling=None, weights='imagenet',
input_shape=(image_size, image_size, 3),
backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils)
if size >= 3:
resnet = ResNet152V2(include_top=False, pooling=None, weights='imagenet',
input_shape=(image_size, image_size, 3), backend=keras.backend,
layers=keras.layers, models=keras.models, utils=keras.utils)
layer_size = model_scale*8*8*8
if is_square(layer_size): # work out layer dimensions
layer_l = int(math.sqrt(layer_size)+0.5)
layer_r = layer_l
else:
layer_m = math.log(math.sqrt(layer_size), 2)
layer_l = 2**math.ceil(layer_m)
layer_r = layer_size // layer_l
layer_l = int(layer_l)
layer_r = int(layer_r)
x_init = None
inp = Input(shape=(image_size, image_size, 3))
x = resnet(inp)
if depth < 0:
depth = 1
if size <= 1:
if size <= 0:
x = Conv2D(model_scale*8, 1, activation=activation)(x) # scale down
x = Reshape((layer_r, layer_l))(x)
else:
x = Conv2D(model_scale*8*4, 1, activation=activation)(x) # scale down a little
x = Reshape((layer_r*2, layer_l*2))(x)
else:
if size == 2:
x = Conv2D(1024, 1, activation=activation)(x) # scale down a bit
x = Reshape((256, 256))(x)
else:
x = Reshape((256, 512))(x) # all weights used
# See https://github.com/OliverRichter/TreeConnect/blob/master/cifar.py
# - TreeConnect inspired layers instead of dense layers.
while depth > 0:
x = LocallyConnected1D(layer_r, 1, activation=activation)(x)
x = Permute((2, 1))(x)
x = LocallyConnected1D(layer_l, 1, activation=activation)(x)
x = Permute((2, 1))(x)
if x_init is not None:
x = Add()([x, x_init]) # add skip connection
x_init = x
depth -= 1
x = Reshape((model_scale, 512))(x) # train against all dlatent values
model = Model(inputs=inp, outputs=x)
model.compile(loss=loss, metrics=[], optimizer=optimizer) # by default: adam optimizer, logcosh used for loss.
return model
def green_block(inp, filters, data_format='channels_first', name=None):
"""
green_block(inp, filters, name=None)
------------------------------------
Implementation of the special residual block used in the paper. The block
consists of two (GroupNorm --> ReLu --> 3x3x3 non-strided Convolution)
units, with a residual connection from the input `inp` to the output. Used
internally in the model. Can be used independently as well.
Parameters
----------
`inp`: An keras.layers.layer instance, required
The keras layer just preceding the green block.
`filters`: integer, required
No. of filters to use in the 3D convolutional block. The output
layer of this green block will have this many no. of channels.
`data_format`: string, optional
The format of the input data. Must be either 'chanels_first' or
'channels_last'. Defaults to `channels_first`, as used in the paper.
`name`: string, optional
The name to be given to this green block. Defaults to None, in which
case, keras uses generated names for the involved layers. If a string
is provided, the names of individual layers are generated by attaching
a relevant prefix from [GroupNorm_, Res_, Conv3D_, Relu_, ], followed
by _1 or _2.
Returns
-------
`out`: A keras.layers.Layer instance
The output of the green block. Has no. of channels equal to `filters`.
The size of the rest of the dimensions remains same as in `inp`.
"""
inp_res = Conv3D(filters=filters,
kernel_size=(1, 1, 1),
strides=1,
data_format=data_format,
name=f'Res_{name}' if name else None)(inp)
# axis=1 for channels_first data format
# No. of groups = 8, as given in the paper
x = GroupNormalization(groups=8,
axis=1 if data_format == 'channels_first' else 0,
name=f'GroupNorm_1_{name}' if name else None)(inp)
x = Activation('relu', name=f'Relu_1_{name}' if name else None)(x)
x = Conv3D(filters=filters,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format=data_format,
name=f'Conv3D_1_{name}' if name else None)(x)
x = GroupNormalization(groups=8,
axis=1 if data_format == 'channels_first' else 0,
name=f'GroupNorm_2_{name}' if name else None)(x)
x = Activation('relu', name=f'Relu_2_{name}' if name else None)(x)
x = Conv3D(filters=filters,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format=data_format,
name=f'Conv3D_2_{name}' if name else None)(x)
out = Add(name=f'Out_{name}' if name else None)([x, inp_res])
return out
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels_in = 3
self.channels_out = 3
self.img_shape_in = (self.img_rows, self.img_cols, self.channels_in)
self.img_shape_out = (self.img_rows, self.img_cols, self.channels_out)
vgg19 = VGG19()
selectedLayers = [4, 5, 7]
selectedOutputs = [vgg19.layers[i].output for i in selectedLayers]
for i in np.arange(len(vgg19.layers)):
vgg19.layers[i].trainable = False
self.lossModel = Model(vgg19.inputs, selectedOutputs)
self.lossModel.summary()
# Configure data loader
self.dataset_name = 'faces_bald_InsNorm_4x4_D2'
self.data_loader = DataLoader()
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
self.discriminator_mask = self.build_discriminator()
self.discriminator_mask.compile(loss='mse',
loss_weights=[2],
optimizer=optimizer,
metrics=['accuracy'])
# -------------------------
# Construct Computational
# Graph of Generator
# -------------------------
# Build the generator
self.generator = self.build_generator()
self.generator.summary()
# Input images and their conditioning images
img_A = Input(shape=self.img_shape_out)
img_B = Input(shape=self.img_shape_in)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# find perceptual loss between fake_A and img_A
fake_A_preproc = Lambda(lambda x: ((x + 1.) * 127.5))(fake_A)
img_A_preproc = Lambda(lambda x: ((x + 1.) * 127.5))(img_A)
fake_A_preproc = Lambda(lambda x: preprocess_input(x))(fake_A_preproc)
img_A_preproc = Lambda(lambda x: preprocess_input(x))(img_A_preproc)
embeddings_fake_A = self.lossModel(fake_A_preproc)
embeddings_img_A = self.lossModel(img_A_preproc)
diffs = []
for emb_fake_A, emb_img_A in zip(embeddings_fake_A, embeddings_img_A):
l2_dist = Lambda(lambda x: L2(x[0], x[1]))([emb_fake_A, emb_img_A])
diffs.append(l2_dist)
diffs = Add()(diffs)
# For the combined model we will only train the generator
self.discriminator.trainable = False
self.discriminator_mask.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
valid_mask = self.discriminator_mask([fake_A, img_B])
def empty_loss(y_true, y_pred):
return y_pred
self.combined = Model(inputs=[img_A, img_B],
outputs=[valid, valid_mask, fake_A, diffs])
self.combined.compile(loss=['mse', 'mse', 'mae', empty_loss],
loss_weights=[1, 2, 1, 0.00001],
optimizer=optimizer)
def simple_resnet_1113(
input_shape=(75, 75, 3), KernelSize=(5, 5), Momentum=0.99):
X_input = Input(input_shape)
#input_CNN = ZeroPadding2D((0, 0))(X_input)
input_CNN = BatchNormalization(momentum=Momentum)(X_input)
## Input Layer
input_CNN = Conv2D(32, kernel_size=KernelSize, padding='same',
name='c11')(input_CNN)
input_CNN = BatchNormalization(momentum=Momentum, name='b11')(input_CNN)
input_CNN = Activation('elu')(input_CNN)
input_CNN = MaxPooling2D((2, 2), strides=(2, 2), name='m11')(input_CNN)
#input_CNN = Dropout(0.25)(input_CNN)
input_CNN = Conv2D(64, kernel_size=KernelSize, padding='same',
name='c12')(input_CNN)
input_CNN = BatchNormalization(momentum=Momentum, name='b12')(input_CNN)
input_CNN = Activation('elu')(input_CNN)
input_CNN = MaxPooling2D((2, 2), strides=(2, 2), name='m12')(input_CNN)
#input_CNN = Dropout(0.25)(input_CNN)
## First Residual
input_CNN_residual = BatchNormalization(momentum=Momentum)(input_CNN)
input_CNN_residual = Conv2D(128, kernel_size=KernelSize,
padding='same')(input_CNN_residual)
input_CNN_residual = BatchNormalization(
momentum=Momentum)(input_CNN_residual)
input_CNN_residual = Activation('elu')(input_CNN_residual)
input_CNN_residual = Dropout(0.25)(input_CNN_residual)
input_CNN_residual = Conv2D(64, kernel_size=KernelSize,
padding='same')(input_CNN_residual)
input_CNN_residual = BatchNormalization(
momentum=Momentum)(input_CNN_residual)
input_CNN_residual = Activation('elu')(input_CNN_residual)
input_CNN_residual = Dropout(0.25)(input_CNN_residual)
input_CNN_residual = Add()([input_CNN_residual, input_CNN])
## Top CNN
top_CNN = Conv2D(128, kernel_size=KernelSize,
padding='same')(input_CNN_residual)
top_CNN = BatchNormalization(momentum=Momentum)(top_CNN)
top_CNN = Activation('elu')(top_CNN)
top_CNN = MaxPooling2D((2, 2), strides=(2, 2))(top_CNN)
top_CNN = Conv2D(256, kernel_size=KernelSize, padding='same')(top_CNN)
top_CNN = BatchNormalization(momentum=Momentum)(top_CNN)
top_CNN = Activation('elu')(top_CNN)
top_CNN = Dropout(0.25)(top_CNN)
top_CNN = MaxPooling2D((2, 2), strides=(2, 2))(top_CNN)
top_CNN = Conv2D(512, kernel_size=KernelSize, padding='same')(top_CNN)
top_CNN = BatchNormalization(momentum=Momentum)(top_CNN)
top_CNN = Activation('elu')(top_CNN)
top_CNN = Dropout(0.25)(top_CNN)
top_CNN = MaxPooling2D((2, 2), strides=(2, 2))(top_CNN)
top_CNN = GlobalMaxPooling2D()(top_CNN)
#Dense Layers
#X = Flatten()(top_CNN)
X = Dense(512)(top_CNN)
X = BatchNormalization(momentum=Momentum)(X)
X = Activation('elu')(X)
X = Dropout(0.25)(X)
X = Dense(256)(X)
X = BatchNormalization(momentum=Momentum)(X)
X = Activation('elu')(X)
X = Dropout(0.25)(X)
X = Dense(1, activation='sigmoid')(X)
model = Model(inputs=X_input, outputs=X, name='simple_resnet')
return model
def build_model(input_shape=(4, 160, 192, 128),
output_channels=3,
weight_L2=0.1,
weight_KL=0.1,
dice_e=1e-8):
"""
build_model(input_shape=(4, 160, 192, 128), output_channels=3, weight_L2=0.1, weight_KL=0.1)
-------------------------------------------
Creates the model used in the BRATS2018 winning solution
by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf)
Parameters
----------
`input_shape`: A 4-tuple, optional.
Shape of the input image. Must be a 4D image of shape (c, H, W, D),
where, each of H, W and D are divisible by 2^4, and c is divisible by 4.
Defaults to the crop size used in the paper, i.e., (4, 160, 192, 128).
`output_channels`: An integer, optional.
The no. of channels in the output. Defaults to 3 (BraTS 2018 format).
`weight_L2`: A real number, optional
The weight to be given to the L2 loss term in the loss function. Adjust to get best
results for your task. Defaults to 0.1.
`weight_KL`: A real number, optional
The weight to be given to the KL loss term in the loss function. Adjust to get best
results for your task. Defaults to 0.1.
`dice_e`: Float, optional
A small epsilon term to add in the denominator of dice loss to avoid dividing by
zero and possible gradient explosion. This argument will be passed to loss_gt function.
Returns
-------
`model`: A keras.models.Model instance
The created model.
"""
c, H, W, D = input_shape
assert len(input_shape) == 4, "Input shape must be a 4-tuple"
assert (c % 4) == 0, "The no. of channels must be divisible by 4"
assert (H % 16) == 0 and (W % 16) == 0 and (D % 16) == 0, \
"All the input dimensions must be divisible by 16"
# -------------------------------------------------------------------------
# Encoder
# -------------------------------------------------------------------------
## Input Layer
inp = Input(input_shape)
## The Initial Block
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_x1')(inp)
## Dropout (0.2)
x = SpatialDropout3D(0.2, data_format='channels_first')(x)
## Green Block x1 (output filters = 32)
x1 = green_block(x, 32, name='x1')
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_32')(x1)
## Green Block x2 (output filters = 64)
x = green_block(x, 64, name='Enc_64_1')
x2 = green_block(x, 64, name='x2')
x = Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_64')(x2)
## Green Blocks x2 (output filters = 128)
x = green_block(x, 128, name='Enc_128_1')
x3 = green_block(x, 128, name='x3')
x = Conv3D(filters=128,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_128')(x3)
## Green Blocks x4 (output filters = 256)
x = green_block(x, 256, name='Enc_256_1')
x = green_block(x, 256, name='Enc_256_2')
x = green_block(x, 256, name='Enc_256_3')
x4 = green_block(x, 256, name='x4')
# -------------------------------------------------------------------------
# Decoder
# -------------------------------------------------------------------------
## GT (Groud Truth) Part
# -------------------------------------------------------------------------
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_128')(x4)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_128')(x)
x = Add(name='Input_Dec_GT_128')([x, x3])
x = green_block(x, 128, name='Dec_GT_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_64')(x)
x = Add(name='Input_Dec_GT_64')([x, x2])
x = green_block(x, 64, name='Dec_GT_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_32')(x)
x = Add(name='Input_Dec_GT_32')([x, x1])
x = green_block(x, 32, name='Dec_GT_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_GT_Output')(x)
### Output Block
out_GT = Conv3D(
filters=output_channels, # No. of tumor classes is 3
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
activation='sigmoid',
name='Dec_GT_Output')(x)
## VAE (Variational Auto Encoder) Part
# -------------------------------------------------------------------------
### VD Block (Reducing dimensionality of the data)
x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4)
x = Activation('relu', name='Dec_VAE_VD_relu')(x)
x = Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Dec_VAE_VD_Conv3D')(x)
# Not mentioned in the paper, but the author used a Flattening layer here.
x = Flatten(name='Dec_VAE_VD_Flatten')(x)
x = Dense(256, name='Dec_VAE_VD_Dense')(x)
### VDraw Block (Sampling)
z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x)
z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x)
x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var])
### VU Block (Upsizing back to a depth of 256)
x = Dense((c // 4) * (H // 16) * (W // 16) * (D // 16))(x)
x = Activation('relu')(x)
x = Reshape(((c // 4), (H // 16), (W // 16), (D // 16)))(x)
x = Conv3D(filters=256,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_256')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_256')(x)
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_128')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_128')(x)
x = green_block(x, 128, name='Dec_VAE_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_64')(x)
x = green_block(x, 64, name='Dec_VAE_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_32')(x)
x = green_block(x, 32, name='Dec_VAE_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_VAE_Output')(x)
### Output Block
out_VAE = Conv3D(filters=4,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_Output')(x)
# Build and Compile the model
out = out_GT
model = Model(inp, outputs=[out, out_VAE]) # Create the model
model.compile(adam(lr=1e-4), [
loss_gt(dice_e),
loss_VAE(input_shape,
z_mean,
z_var,
weight_L2=weight_L2,
weight_KL=weight_KL)
],
metrics=[dice_coefficient])
return model
def get_lattice_max(cnn_all):
maxed_cnn_all = [tf.reduce_max(t, axis=-2) for t in cnn_all]
result = Add()(maxed_cnn_all)
return result
def get_model(wordMatrix, model_param=None):
if model_param is None:
model_param = {'layer_num': 1}
if not 'pooling_mode' in model_param:
model_param['pooling_mode'] = 'gat_2' #'gcn'
if not 'if_d' in model_param:
model_param['if_d'] = True
if not 'n_param' in model_param:
model_param['n_param'] = 1024
# print(json.dumps(model_param))
# {"layer_num": 1, "pooling_mode": "gcn", "if_d": true}
qu_l = 50
pre_l = 20
pad_l = 5
""" inputs """
qu_ids = Input(shape=(qu_l, ), dtype='int32', name='qu')
pre_ids = Input(shape=(pre_l, ), dtype='int32', name='pre')
input_layers = [qu_ids, pre_ids]
# (?, 50) (?, 20)
qu_mid_r = Input(shape=(qu_l, ), dtype='int32', name='qu_mid')
qu_head = Input(shape=(qu_l, pad_l), dtype='int32', name='qu_head')
qu_tail = Input(shape=(qu_l, pad_l), dtype='int32', name='qu_tail')
pre_mid_r = Input(shape=(pre_l, ), dtype='int32', name='pre_mid')
pre_head = Input(shape=(pre_l, pad_l), dtype='int32', name='pre_head')
pre_tail = Input(shape=(pre_l, pad_l), dtype='int32', name='pre_tail')
qu_i_ids = [qu_head, qu_mid_r, qu_tail]
pre_i_ids = [pre_head, pre_mid_r, pre_tail]
input_layers += [qu_head, qu_mid_r, qu_tail, pre_head, pre_mid_r, pre_tail]
expand_dim = Lambda(lambda x: K.expand_dims(x, -1),
output_shape=lambda x: x + (1, ))
qu_i_ids[1] = expand_dim(qu_i_ids[1])
pre_i_ids[1] = expand_dim(pre_i_ids[1])
# Tensor("lambda_1/ExpandDims:0", shape=(?, 50, 1), dtype=int32)
# Tensor("lambda_1_1/ExpandDims:0", shape=(?, 20, 1), dtype=int32)
""" embedding """
input_dim, output_dim = np.shape(wordMatrix)
embedding = Embedding(input_dim,
output_dim,
weights=[wordMatrix],
name='embedding')
qu_words = embedding(qu_ids)
pre_words = embedding(pre_ids)
# (?, 50, 300) (?, 20, 300)
""" convolution """
n_param = model_param['n_param'] #1024
cnn1_head = Dense(n_param, name='cnn1_head')
cnn1_mid = Dense(n_param,
name='cnn1_mid') if model_param['if_d'] else cnn1_head
cnn1_tail = Dense(n_param,
name='cnn1_tail') if model_param['if_d'] else cnn1_head
cnn1 = [cnn1_head, cnn1_mid, cnn1_tail]
gate_dense = None
if model_param['pooling_mode'] == 'gat':
base_gate = Dense(1, name='gate1_head', activation='sigmoid')
gate_dense = [
base_gate,
Dense(1, name='gate1_mid', activation='sigmoid')
if model_param['if_d'] else base_gate,
Dense(1, name='gate1_tail', activation='sigmoid')
if model_param['if_d'] else base_gate,
]
elif model_param['pooling_mode'] in ['gat_n', 'gat_2']:
base_gate = Dense(1, name='gate1_head', activation='sigmoid')
gate_dense = [
base_gate,
Dense(1, name='gate1_mid', activation=None)
if model_param['if_d'] else base_gate,
Dense(1, name='gate1_tail', activation=None)
if model_param['if_d'] else base_gate,
]
qu_results = gcn_layer(qu_i_ids,
qu_words,
cnn1,
mode=model_param['pooling_mode'],
gate_dense=gate_dense)
pre_results = gcn_layer(pre_i_ids,
pre_words,
cnn1,
mode=model_param['pooling_mode'],
gate_dense=gate_dense)
# (?, 80, 1024)
# (?, 160, 1024)
for i in range(model_param['layer_num'] - 1):
cnn2_head = Dense(n_param, name='cnn%d_head' % (i + 2))
cnn2_mid = Dense(n_param, name='cnn%d_mid' %
(i + 2)) if model_param['if_d'] else cnn2_head
cnn2_tail = Dense(n_param, name='cnn%d_tail' %
(i + 2)) if model_param['if_d'] else cnn2_head
cnn2 = [cnn2_head, cnn2_mid, cnn2_tail]
gate_dense_2 = None
if model_param['pooling_mode'] == 'gat':
base_gate_2 = Dense(1,
name='gate%d_head' % (i + 2),
activation='sigmoid')
gate_dense_2 = [
base_gate_2,
Dense(1, name='gate%d_mid' % (i + 2), activation='sigmoid')
if model_param['if_d'] else base_gate_2,
Dense(1, name='gate%d_tail' % (i + 2), activation='sigmoid')
if model_param['if_d'] else base_gate_2,
]
elif model_param['pooling_mode'] in ['gat_n', 'gat_2']:
base_gate_2 = Dense(1,
name='gate%d_head' % (i + 2),
activation='sigmoid')
gate_dense_2 = [
base_gate_2,
Dense(1, name='gate%d_mid' % (i + 2), activation=None)
if model_param['if_d'] else base_gate_2,
Dense(1, name='gate%d_tail' % (i + 2), activation=None)
if model_param['if_d'] else base_gate_2,
]
qu_results_2 = gcn_layer(qu_i_ids,
qu_results,
cnn2,
active=None,
mode=model_param['pooling_mode'],
gate_dense=gate_dense_2)
pre_results_2 = gcn_layer(pre_i_ids,
pre_results,
cnn2,
active=None,
mode=model_param['pooling_mode'],
gate_dense=gate_dense_2)
# (?, 80, 1024)
# (?, 160, 1024)
qu_results_2 = Activation('relu')(Add()([qu_results, qu_results_2]))
pre_results_2 = Activation('relu')(Add()([pre_results, pre_results_2]))
qu_results = qu_results_2
pre_results = pre_results_2
""" merge """
label = count_similarity(qu_results, pre_results)
# (?, 1)
model = Model(inputs=input_layers, outputs=[label])
model.compile(loss={'label': 'binary_crossentropy'}, optimizer='adadelta')
return model
def share_stream(x_shape, outputNum=256):
input = Input(x_shape)
x_a = input
conv1 = Conv2D(filters=32,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
conv1 = Activation('relu')(conv1)
conv1 = Conv2D(filters=32,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(conv1)
x_a = Conv2D(filters=32,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = add([x_a, conv1])
attention = Conv2D(filters=32,
kernel_size=(5, 5),
activation='sigmoid',
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = multiply([x_a, attention])
x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a)
conv1 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
conv1 = Activation('relu')(conv1)
conv1 = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(conv1)
x_a = Conv2D(filters=64,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = add([x_a, conv1])
x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a)
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
temp = Multiply()([
Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
trainable=Trainable)(Lambda(lambda x: x * 10000000.)(
Activation("tanh")(Lambda(lambda x: x / 10000000.)(x)))), x
])
for jj in range(2, taylor):
temp = Add()([
Multiply()([
Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
trainable=Trainable)(Lambda(lambda x: x * 10000000.)(
Activation("tanh")(
Lambda(lambda x: x / 10000000.)(x)))),
Lambda(lambda x: (x**jj) / math.factorial(jj))(x)
]), temp
])
temp = Add()([
Conv2D(filters=128,
kernel_size=(3, 3),
padding='same',
trainable=Trainable)(Lambda(lambda x: x * 10000000.)(
Activation("tanh")(Lambda(lambda x: x / 10000000.)(x)))),
temp
])
conv2 = Lambda(lambda x: x * 10000000.)(Activation("tanh")(
Lambda(lambda x: x / 10000000.)(temp)))
conv2 = Conv2D(filters=128,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(conv2)
x_a = Conv2D(filters=128,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = add([x_a, conv2])
x_a = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')(x_a)
conv3 = Conv2D(filters=outputNum,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
conv3 = Activation('relu')(conv3)
conv3 = Conv2D(filters=outputNum,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(conv3)
x_a = Conv2D(filters=outputNum,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
use_bias=use_bias,
trainable=Trainable)(x_a)
x_a = add([x_a, conv3])
x_a = GlobalMaxPooling2D()(x_a)
shared_layer = Model(input, x_a)
return shared_layer
def dense_gener(self):
def residual_block(layer_input, filters):
"""Residual block described in paper"""
d = Conv2D(filters, kernel_size=(3, 3), strides=1,
padding='same')(layer_input)
d = Activation('relu')(d)
# d = BatchNormalization(momentum=0.8)(d)
d = Conv2D(filters, kernel_size=(3, 3), strides=1,
padding='same')(d)
d = Activation('relu')(d)
# d = BatchNormalization(momentum=0.8)(d)
d = Add()([d, layer_input])
return d
fil = 1
# Low resolution image input
img_lr = Input(shape=self.lr_shape)
# Pre-residual block
c0 = Conv2D(fil, kernel_size=3, strides=1, padding='same')(img_lr)
c0 = LeakyReLU(alpha=0.2)(c0)
# e0 = Conv2D(1, kernel_size=3, strides=1, padding='same')(c0)
# e0 = BatchNormalization(momentum=0.8)(e0)
# e0 = LeakyReLU(alpha=0.2)(e0)
rr1 = residual_block(c0, fil)
for _ in range(self.n_residual_blocks - 1):
rr1 = residual_block(rr1, fil)
rr2 = Add()([c0, rr1])
rr2 = residual_block(rr2, fil)
for _ in range(self.n_residual_blocks - 1):
rr2 = residual_block(rr2, fil)
rr3 = Add()([rr1, rr2])
rr3 = residual_block(rr3, fil)
for _ in range(self.n_residual_blocks - 1):
rr3 = residual_block(rr3, fil)
# l0 = Conv2D(16, kernel_size=9, strides=1, padding='same')(img_lr)
# l0 = LeakyReLU(alpha=0.2)(l0)
#
# e1 = Conv2D(16, kernel_size=3, strides=1, padding='same')(l0)
# e1 = BatchNormalization(momentum=0.8)(e1)
# e1 = LeakyReLU(alpha=0.2)(e1)
#
# rr2 = residual_block(e1, fil)
# for _ in range(self.n_residual_blocks - 1):
# rr2 = residual_block(rr2, fil)
#
# ll0 = Conv2D(16, kernel_size=27, strides=1, padding='same')(img_lr)
# ll0 = LeakyReLU(alpha=0.2)(ll0)
#
# ee1 = Conv2D(16, kernel_size=3, strides=1, padding='same')(ll0)
# ee1 = BatchNormalization(momentum=0.8)(ee1)
# ee1 = LeakyReLU(alpha=0.2)(ee1)
# rr3 = residual_block(ee1, fil)
# for _ in range(self.n_residual_blocks - 1):
# rr3 = residual_block(rr3, fil)
dens0 = Concatenate()([rr3, rr2, rr1, c0, img_lr])
# for _ in range(self.n_residual_blocks - 1):
# e0 = residual_block(e0, self.gf)
# fusion
f0 = Conv2D(64,
kernel_size=(3, self.lr_width),
strides=1,
padding="same")(dens0)
f0 = Activation('relu')(f0)
# f0 = BatchNormalization(momentum=0.8)(f0)
# f0 = Reshape((128,128,64))(f0)
# r = residual_block(f0, self.gf)
# for _ in range(self.n_residual_blocks - 1):
# r = residual_block(r, self.gf)
# f1 = Add()([f0,r])
# c1 = Conv2D(64, kernel_size=(3, 3), strides=1, padding='same')(f0)
# c1 = Activation('relu')(c1)
# c1 = BatchNormalization(momentum=0.8)(c1)
# Propogate through residual blocks
# r = residual_block(c1, self.gf)
# for _ in range(self.n_residual_blocks - 1):
# r = residual_block(r, self.gf)
# ,,,,
# Post-residual block
# c2 = Conv2D(64, kernel_size=(3, 3), strides=1, padding='same')(f0)
# c2 = Activation('relu')(c2)
# c2 = BatchNormalization(momentum=0.8)(c2)
#
# d0 = Conv2D(32, kernel_size=3, strides=1, padding="same")(c2)
# d0 = Activation('relu')(d0)
# d0 = BatchNormalization(momentum=0.8)(d0)
# ,,,,
# generate
#
# g2 = Conv2D(128, kernel_size=3, strides=1, padding="same")(d0)
# g2 = BatchNormalization(momentum=0.8)(g2)
# g2 = Activation('relu')(g2)
#
# g3 = Conv2D(128, kernel_size=3, strides=1, padding="same")(g2)
# g3 = BatchNormalization(momentum=0.8)(g3)
# g3 = Activation('relu')(g3)
# Generate high resolution output
gen_hr = Conv2D(self.channels,
kernel_size=3,
strides=1,
padding='same',
activation='tanh')(f0)
return Model(img_lr, gen_hr)
def get_crfrnn_model_def():
"""
Returns Keras CRN-RNN model definition.
Currently, only 500 x 500 images are supported. However, one can get this to
work with different image sizes by adjusting the parameters of the Cropping2D layers
below.
"""
channels, height, weight = 3, 500, 500
# Input
input_shape = (height, weight, 3)
img_input = Input(shape=input_shape)
# Add plenty of zero padding
x = ZeroPadding2D(padding=(100, 100))(img_input)
# VGG-16 convolution block 1
x = Conv2D(64, (3, 3), activation='relu', padding='valid',
name='conv1_1')(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same',
name='conv1_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x)
# VGG-16 convolution block 2
x = Conv2D(128, (3, 3), activation='relu', padding='same',
name='conv2_1')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same',
name='conv2_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2', padding='same')(x)
# VGG-16 convolution block 3
x = Conv2D(256, (3, 3), activation='relu', padding='same',
name='conv3_1')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same',
name='conv3_2')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same',
name='conv3_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3', padding='same')(x)
pool3 = x
# VGG-16 convolution block 4
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv4_1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv4_2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv4_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4', padding='same')(x)
pool4 = x
# VGG-16 convolution block 5
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv5_1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv5_2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv5_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5', padding='same')(x)
# Fully-connected layers converted to convolution layers
x = Conv2D(4096, (7, 7), activation='relu', padding='valid', name='fc6')(x)
x = Dropout(0.5)(x)
x = Conv2D(4096, (1, 1), activation='relu', padding='valid', name='fc7')(x)
x = Dropout(0.5)(x)
x = Conv2D(21, (1, 1), padding='valid', name='score-fr')(x)
# Deconvolution
score2 = Conv2DTranspose(21, (4, 4), strides=2, name='score2')(x)
# Skip connections from pool4
score_pool4 = Conv2D(21, (1, 1), name='score-pool4')(pool4)
score_pool4c = Cropping2D((5, 5))(score_pool4)
score_fused = Add()([score2, score_pool4c])
score4 = Conv2DTranspose(21, (4, 4),
strides=2,
name='score4',
use_bias=False)(score_fused)
# Skip connections from pool3
score_pool3 = Conv2D(21, (1, 1), name='score-pool3')(pool3)
score_pool3c = Cropping2D((9, 9))(score_pool3)
# Fuse things together
score_final = Add()([score4, score_pool3c])
# Final up-sampling and cropping
upsample = Conv2DTranspose(21, (16, 16),
strides=8,
name='upsample',
use_bias=False)(score_final)
upscore = Cropping2D(((31, 37), (31, 37)))(upsample)
output = CrfRnnLayer(image_dims=(height, weight),
num_classes=21,
theta_alpha=160.,
theta_beta=3.,
theta_gamma=3.,
num_iterations=10,
name='crfrnn')([upscore, img_input])
# Build the model
model = Model(img_input, output, name='crfrnn_net')
return model
def build_generator(self):
def residual_block(layer_input, filters):
"""Residual block described in paper"""
d = Conv2D(filters, kernel_size=3, strides=1,
padding='same')(layer_input)
d = Activation('relu')(d)
d = BatchNormalization(momentum=0.8)(d)
d = Conv2D(filters, kernel_size=3, strides=1, padding='same')(d)
d = BatchNormalization(momentum=0.8)(d)
d = Add()([d, layer_input])
return d
def deconv2d(layer_input):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(256, kernel_size=3, strides=1, padding='same')(u)
u = Activation('relu')(u)
return u
# Low resolution image input
img_lr = Input(shape=self.lr_shape)
# Pre-residual block
c0 = Conv2D(64, kernel_size=9, strides=1, padding='same')(img_lr)
c0 = Activation('relu')(c0)
d1 = Conv2D(64,
kernel_size=(3, self.lr_width),
strides=1,
padding="same")(c0)
d1 = Activation('relu')(d1)
c1 = Conv2D(64, kernel_size=3, strides=1, padding='same')(d1)
c1 = Activation('relu')(c1)
# Propogate through residual blocks
r = residual_block(c1, self.gf)
for _ in range(self.n_residual_blocks - 1):
r = residual_block(r, self.gf)
# Post-residual block
c2 = Conv2D(64, kernel_size=3, strides=1, padding='same')(r)
c2 = Activation('relu')(c2)
c2 = BatchNormalization(momentum=0.8)(c2)
c2 = Add()([c2, c1])
# Upsampling
# u1 = deconv2d(c2)
# u2 = deconv2d(u1)
u1 = Conv2D(256, kernel_size=3, strides=1, padding="same")(c2)
u1 = Activation('relu')(u1)
u2 = Conv2D(256, kernel_size=3, strides=1, padding="same")(u1)
u2 = Activation('relu')(u2)
# Generate high resolution output
gen_hr = Conv2D(self.channels,
kernel_size=9,
strides=1,
padding='same',
activation='tanh')(u2)
return Model(img_lr, gen_hr)
def create_model(self,
shape,
stddev=0.1,
skip_shortcuts=False,
deconvolutional=False,
loss='mse'):
'''
Skip shortcuts: https://arxiv.org/pdf/1606.08921.pdf
'''
inputs = Input(shape=shape)
inputs_noizy = inputs
inputs_noizy = GaussianNoise(stddev)(inputs_noizy)
x1 = Conv2D(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(inputs_noizy)
x2 = Conv2D(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x1)
x2 = BatchNormalization()(x2)
x2 = Dropout(0.2)(x2)
x3 = Conv2D(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x2)
x4 = Conv2D(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x3)
x4 = BatchNormalization()(x4)
x4 = Dropout(0.2)(x4)
if deconvolutional == False:
x5 = Conv2D(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x4)
x6 = Conv2D(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x5)
x6 = BatchNormalization()(x6)
x6 = Dropout(0.2)(x6)
x7 = Conv2D(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x6)
x8 = Conv2D(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x7)
x8 = BatchNormalization()(x8)
x8 = Dropout(0.2)(x8)
outputs = Conv2D(3,
self.filter_size,
padding='same',
activation='sigmoid')(x8)
else:
x5 = Conv2DTranspose(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x4)
x6 = Conv2DTranspose(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x5)
x6 = BatchNormalization()(x6)
x6 = Dropout(0.2)(x6)
x7 = Conv2DTranspose(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x6)
x8 = Conv2DTranspose(self.kernel_size,
self.filter_size,
padding='same',
activation='relu')(x7)
x8 = BatchNormalization()(x8)
x8 = Dropout(0.2)(x8)
outputs = Conv2DTranspose(shape[-1],
self.filter_size,
padding='same',
activation='sigmoid')(x8)
if skip_shortcuts == True:
y = BatchNormalization()(inputs)
y = LeakyReLU()(y)
outputs = Add([outputs, y])
model = Model(inputs=inputs, outputs=outputs)
model.summary()
model.compile(loss=loss,
optimizer=Adam(lr=self.learn_rate),
metrics=['accuracy'])
return model
def convolutional_block(X, f, filters, stage, block, s=2):
"""
Implementation of the convolutional block as defined in Figure 4
Arguments:
X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
f -- integer, specifying the shape of the middle CONV's window for the main path
filters -- python list of integers, defining the number of filters in the CONV layers of the main path
stage -- integer, used to name the layers, depending on their position in the network
block -- string/character, used to name the layers, depending on their position in the network
s -- Integer, specifying the stride to be used
Returns:
X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)
"""
# defining name basis
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
# Retrieve Filters
F1, F2, F3 = filters
# Save the input value
X_shortcut = X
##### MAIN PATH #####
# First component of main path
X = Conv2D(F1, (1, 1),
strides=(s, s),
name=conv_name_base + '2a',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)
X = Activation('relu')(X)
### START CODE HERE ###
# Second component of main path (≈3 lines)
X = Conv2D(F2, (f, f),
strides=(1, 1),
padding='same',
name=conv_name_base + '2b',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)
X = Activation('relu')(X)
# Third component of main path (≈2 lines)
X = Conv2D(F3, (1, 1),
strides=(1, 1),
padding='valid',
name=conv_name_base + '2c',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X)
##### SHORTCUT PATH #### (≈2 lines)
X_shortcut = Conv2D(F3, (1, 1),
strides=(s, s),
padding='valid',
name=conv_name_base + '1',
kernel_initializer=glorot_uniform(seed=0))(X_shortcut)
X_shortcut = BatchNormalization(axis=3,
name=bn_name_base + '1')(X_shortcut)
# Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)
X = Add()([X, X_shortcut])
X = Activation('relu')(X)
### END CODE HERE ###
return X
def get_predict_model(config):
h, w = config.IMAGE_SHAPE[:2]
if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6):
raise Exception("Image size must be dividable by 2 at least 6 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# 输入进来的图片必须是2的6次方以上的倍数
input_image = Input(shape=[None, None, config.IMAGE_SHAPE[2]],
name="input_image")
# meta包含了一些必要信息
input_image_meta = Input(shape=[config.IMAGE_META_SIZE],
name="input_image_meta")
# 输入进来的先验框
input_anchors = Input(shape=[None, 4], name="input_anchors")
# 获得Resnet里的压缩程度不同的一些层
_, C2, C3, C4, C5 = get_resnet(input_image,
stage5=True,
train_bn=config.TRAIN_BN)
# 组合成特征金字塔的结构
# P5长宽共压缩了5次
# Height/32,Width/32,256
P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5)
# P4长宽共压缩了4次
# Height/16,Width/16,256
P4 = Add(name="fpn_p4add")([
UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4)
])
# P4长宽共压缩了3次
# Height/8,Width/8,256
P3 = Add(name="fpn_p3add")([
UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3)
])
# P4长宽共压缩了2次
# Height/4,Width/4,256
P2 = Add(name="fpn_p2add")([
UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2)
])
# 各自进行一次256通道的卷积，此时P2、P3、P4、P5通道数相同
# Height/4,Width/4,256
P2 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p2")(P2)
# Height/8,Width/8,256
P3 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p3")(P3)
# Height/16,Width/16,256
P4 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p4")(P4)
# Height/32,Width/32,256
P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p5")(P5)
# 在建议框网络里面还有一个P6用于获取建议框
# Height/64,Width/64,256
P6 = MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5)
# P2, P3, P4, P5, P6可以用于获取建议框
rpn_feature_maps = [P2, P3, P4, P5, P6]
# P2, P3, P4, P5用于获取mask信息
mrcnn_feature_maps = [P2, P3, P4, P5]
anchors = input_anchors
# 建立RPN模型
rpn = build_rpn_model(len(config.RPN_ANCHOR_RATIOS),
config.TOP_DOWN_PYRAMID_SIZE)
rpn_class_logits, rpn_class, rpn_bbox = [], [], []
# 获得RPN网络的预测结果，进行格式调整，把五个特征层的结果进行堆叠
for p in rpn_feature_maps: # [P2, P3, P4, P5, P6]
logits, classes, bbox = rpn([p])
rpn_class_logits.append(logits)
rpn_class.append(classes)
rpn_bbox.append(bbox)
rpn_class_logits = Concatenate(axis=1,
name="rpn_class_logits")(rpn_class_logits)
rpn_class = Concatenate(axis=1, name="rpn_class")(rpn_class)
rpn_bbox = Concatenate(axis=1, name="rpn_bbox")(rpn_bbox)
# 此时获得的rpn_class_logits、rpn_class、rpn_bbox的维度是
# rpn_class_logits : Batch_size, num_anchors, 2
# rpn_class : Batch_size, num_anchors, 2
# rpn_bbox : Batch_size, num_anchors, 4
proposal_count = config.POST_NMS_ROIS_INFERENCE
# Batch_size, proposal_count, 4
# 对先验框进行解码
rpn_rois = ProposalLayer(proposal_count=proposal_count,
nms_threshold=config.RPN_NMS_THRESHOLD,
name="ROI",
config=config)([rpn_class, rpn_bbox, anchors])
# 获得classifier的结果
mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\
fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, input_image_meta,
config.POOL_SIZE, config.NUM_CLASSES,
train_bn=config.TRAIN_BN,
fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)
# 最终预测框
detections = DetectionLayer(
config, name="mrcnn_detection"
)( # RPN层的rpn_rois建议框, mrcnn_class, mrcnn_bbox, input_image_meta
[rpn_rois, mrcnn_class, mrcnn_bbox, input_image_meta])
detection_boxes = Lambda(lambda x: x[..., :4])(detections)
# 获得mask的结果
mrcnn_mask = build_fpn_mask_graph(detection_boxes,
mrcnn_feature_maps,
input_image_meta,
config.MASK_POOL_SIZE,
config.NUM_CLASSES,
train_bn=config.TRAIN_BN)
# 汇总，作为输出
model = Model([input_image, input_image_meta, input_anchors], [
detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, rpn_class,
rpn_bbox
],
name='mask_rcnn')
return model
for i in range(0, 1): #num of blocks
for k in range(
0, 9
): #num of dilations/resolutions (increase them to get RF > sample_len for the encoder)
z = x
x = Convolution1D(512, 1, padding='same')(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
x = DepthwiseConv1D(x, 3, dilation_rate=2**k)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
t = Convolution1D(128, 1, padding='same')(x)
if (i != 0) or (k != 0):
s = Add()(
[t, s]
) #skip connection, gradually adding the masks from each dilated block
else:
s = t
if (i != 2) or (k != 8):
x = Convolution1D(128, 1, padding='same')(
x) #1x1 conv to reshape the feature map.
x = Add()(
[z, x]) #output connection, with a residual block to the input
x = LeakyReLU()(s)
x = Convolution1D(512, 1, padding='same', activation='sigmoid')(x)
x = Multiply()([e, x]) #multiplies the masks
t = Conv1DTranspose(x, 1, 16, strides=8, padding='same')
def get_train_model(config):
h, w = config.IMAGE_SHAPE[:2]
if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6):
raise Exception("Image size must be dividable by 2 at least 6 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# 输入进来的图片必须是2的6次方以上的倍数
input_image = Input(shape=[None, None, config.IMAGE_SHAPE[2]],
name="input_image")
# meta包含了一些必要信息
input_image_meta = Input(shape=[config.IMAGE_META_SIZE],
name="input_image_meta")
# RPN建议框网络的真实框信息
input_rpn_match = Input(shape=[None, 1],
name="input_rpn_match",
dtype=tf.int32)
input_rpn_bbox = Input(shape=[None, 4],
name="input_rpn_bbox",
dtype=tf.float32)
# 种类信息
input_gt_class_ids = Input(shape=[None],
name="input_gt_class_ids",
dtype=tf.int32)
# 框的位置信息
input_gt_boxes = Input(shape=[None, 4],
name="input_gt_boxes",
dtype=tf.float32)
# 标准化到0-1之间
gt_boxes = Lambda(lambda x: norm_boxes_graph(x,
K.shape(input_image)[1:3]))(
input_gt_boxes)
# mask语义分析信息
# [batch, height, width, MAX_GT_INSTANCES]
if config.USE_MINI_MASK:
input_gt_masks = Input(
shape=[config.MINI_MASK_SHAPE[0], config.MINI_MASK_SHAPE[1], None],
name="input_gt_masks",
dtype=bool)
else:
input_gt_masks = Input(
shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None],
name="input_gt_masks",
dtype=bool)
# 获得Resnet里的压缩程度不同的一些层
_, C2, C3, C4, C5 = get_resnet(input_image,
stage5=True,
train_bn=config.TRAIN_BN)
# 组合成特征金字塔的结构
# P5长宽共压缩了5次
# Height/32,Width/32,256
P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c5p5')(C5)
# P4长宽共压缩了4次
# Height/16,Width/16,256
P4 = Add(name="fpn_p4add")([
UpSampling2D(size=(2, 2), name="fpn_p5upsampled")(P5),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c4p4')(C4)
])
# P4长宽共压缩了3次
# Height/8,Width/8,256
P3 = Add(name="fpn_p3add")([
UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c3p3')(C3)
])
# P4长宽共压缩了2次
# Height/4,Width/4,256
P2 = Add(name="fpn_p2add")([
UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1), name='fpn_c2p2')(C2)
])
# 各自进行一次256通道的卷积，此时P2、P3、P4、P5通道数相同
# Height/4,Width/4,256
P2 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p2")(P2)
# Height/8,Width/8,256
P3 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p3")(P3)
# Height/16,Width/16,256
P4 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p4")(P4)
# Height/32,Width/32,256
P5 = Conv2D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3),
padding="SAME",
name="fpn_p5")(P5)
# 在建议框网络里面还有一个P6用于获取建议框
# Height/64,Width/64,256
P6 = MaxPooling2D(pool_size=(1, 1), strides=2, name="fpn_p6")(P5)
# P2, P3, P4, P5, P6可以用于获取建议框
rpn_feature_maps = [P2, P3, P4, P5, P6]
# P2, P3, P4, P5用于获取mask信息
mrcnn_feature_maps = [P2, P3, P4, P5]
anchors = get_anchors(config, config.IMAGE_SHAPE)
# 拓展anchors的shape，第一个维度拓展为batch_size
anchors = np.broadcast_to(anchors, (config.BATCH_SIZE, ) + anchors.shape)
# 将anchors转化成tensor的形式
anchors = Lambda(lambda x: tf.Variable(anchors),
name="anchors")(input_image)
# 建立RPN模型
rpn = build_rpn_model(len(config.RPN_ANCHOR_RATIOS),
config.TOP_DOWN_PYRAMID_SIZE)
rpn_class_logits, rpn_class, rpn_bbox = [], [], []
# 获得RPN网络的预测结果，进行格式调整，把五个特征层的结果进行堆叠
for p in rpn_feature_maps:
logits, classes, bbox = rpn([p])
rpn_class_logits.append(logits)
rpn_class.append(classes)
rpn_bbox.append(bbox)
rpn_class_logits = Concatenate(axis=1,
name="rpn_class_logits")(rpn_class_logits)
rpn_class = Concatenate(axis=1, name="rpn_class")(rpn_class)
rpn_bbox = Concatenate(axis=1, name="rpn_bbox")(rpn_bbox)
# 此时获得的rpn_class_logits、rpn_class、rpn_bbox的维度是
# rpn_class_logits : Batch_size, num_anchors, 2
# rpn_class : Batch_size, num_anchors, 2
# rpn_bbox : Batch_size, num_anchors, 4
proposal_count = config.POST_NMS_ROIS_TRAINING
# Batch_size, proposal_count, 4
rpn_rois = ProposalLayer(proposal_count=proposal_count,
nms_threshold=config.RPN_NMS_THRESHOLD,
name="ROI",
config=config)([rpn_class, rpn_bbox, anchors])
active_class_ids = Lambda(lambda x: parse_image_meta_graph(x)[
"active_class_ids"])(input_image_meta)
if not config.USE_RPN_ROIS:
# 使用外部输入的建议框
input_rois = Input(shape=[config.POST_NMS_ROIS_TRAINING, 4],
name="input_roi",
dtype=np.int32)
# Normalize coordinates
target_rois = Lambda(
lambda x: norm_boxes_graph(x,
K.shape(input_image)[1:3]))(input_rois)
else:
# 利用预测到的建议框进行下一步的操作
target_rois = rpn_rois
"""找到建议框的ground_truth
Inputs:
proposals: [batch, N, (y1, x1, y2, x2)]建议框
gt_class_ids: [batch, MAX_GT_INSTANCES]每个真实框对应的类
gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)]真实框的位置
gt_masks: [batch, height, width, MAX_GT_INSTANCES]真实框的语义分割情况
Returns:
rois: [batch, TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)]内部真实存在目标的建议框
target_class_ids: [batch, TRAIN_ROIS_PER_IMAGE]每个建议框对应的类
target_deltas: [batch, TRAIN_ROIS_PER_IMAGE, (dy, dx, log(dh), log(dw)]每个建议框应该有的调整参数
target_mask: [batch, TRAIN_ROIS_PER_IMAGE, height, width]每个建议框语义分割情况
"""
rois, target_class_ids, target_bbox, target_mask =\
DetectionTargetLayer(config, name="proposal_targets")([
target_rois, input_gt_class_ids, gt_boxes, input_gt_masks])
# 找到合适的建议框的classifier预测结果
mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\
fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta,
config.POOL_SIZE, config.NUM_CLASSES,
train_bn=config.TRAIN_BN,
fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)
# 找到合适的建议框的mask预测结果
mrcnn_mask = build_fpn_mask_graph(rois,
mrcnn_feature_maps,
input_image_meta,
config.MASK_POOL_SIZE,
config.NUM_CLASSES,
train_bn=config.TRAIN_BN)
output_rois = Lambda(lambda x: x * 1, name="output_rois")(rois)
# Losses
rpn_class_loss = Lambda(lambda x: rpn_class_loss_graph(*x),
name="rpn_class_loss")(
[input_rpn_match, rpn_class_logits])
rpn_bbox_loss = Lambda(lambda x: rpn_bbox_loss_graph(config, *x),
name="rpn_bbox_loss")(
[input_rpn_bbox, input_rpn_match, rpn_bbox])
class_loss = Lambda(lambda x: mrcnn_class_loss_graph(*x),
name="mrcnn_class_loss")([
target_class_ids, mrcnn_class_logits,
active_class_ids
])
bbox_loss = Lambda(lambda x: mrcnn_bbox_loss_graph(*x),
name="mrcnn_bbox_loss")(
[target_bbox, target_class_ids, mrcnn_bbox])
mask_loss = Lambda(lambda x: mrcnn_mask_loss_graph(*x),
name="mrcnn_mask_loss")(
[target_mask, target_class_ids, mrcnn_mask])
# Model
inputs = [
input_image, input_image_meta, input_rpn_match, input_rpn_bbox,
input_gt_class_ids, input_gt_boxes, input_gt_masks
]
if not config.USE_RPN_ROIS:
inputs.append(input_rois)
outputs = [
rpn_class_logits, rpn_class, rpn_bbox, mrcnn_class_logits, mrcnn_class,
mrcnn_bbox, mrcnn_mask, rpn_rois, output_rois, rpn_class_loss,
rpn_bbox_loss, class_loss, bbox_loss, mask_loss
]
model = Model(inputs, outputs, name='mask_rcnn')
return model
def encoder_basic_block(
self,
input,
out_filters,
kernel_size=3,
strides=1,
padding='same',
bias=False,
name=''
):
"""Creates a basic encoder block.
Main brach architecture:
1. Conv2D
2. BatchNormalization
3. ReLU
4. Conv2D
5. BatchNormalization
Residual branch architecture:
1. Conv2D, if `strides` is greater than 1
The output of the main and residual branches are then added together
with ReLU activation.
Args:
input (tensor): A tensor or variable.
out_filters (int): The number of filters in the block output.
kernel_size (int, tuple, list, optional): A tuple/list of 2
integers, specifying the height and width of the 2D kernel
window. In case it's a single integer, it's value is used
for all spatial dimensions. Default: 3.
strides (int, tuple, list, optional): A tuple/list of 2
integers, specifying the strides along the height and width
of the 2D input. In case it's a single integer, it's value
is used for all spatial dimensions. Default: 1.
padding (str, optional): One of "valid" or "same" (case-insensitive).
Default: "same".
bias (bool, optional): If ``True``, adds a learnable bias.
Default: ``False``.
name (string, optional): A string to identify this block.
Default: Empty string.
Returns:
The output tensor of the block.
"""
residual = input
x = Conv2D(
out_filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
use_bias=bias,
name=name + '/main/conv2d_1'
)(input)
x = BatchNormalization(name=name + '/main/bn_1')(x)
x = Activation('relu', name=name + '/main/relu_1')(x)
x = Conv2D(
out_filters,
kernel_size=kernel_size,
strides=1,
padding=padding,
use_bias=bias,
name=name + '/main/conv2d_2'
)(x)
x = BatchNormalization(name=name + '/main/bn_2')(x)
if strides > 1:
residual = Conv2D(
out_filters,
kernel_size=1,
strides=strides,
padding=padding,
use_bias=bias,
name=name + '/res/conv2d_1'
)(residual)
residual = BatchNormalization(name=name + '/res/bn_1')(residual)
x = Add(name=name + '/add')([x, residual])
x = Activation('relu', name=name + '/relu_1')(x)
return x
def identity_block_3D(X, f, filters, stage, block):
"""
Implementation of the identity block as defined in Figure 3
Arguments:
X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
f -- integer, specifying the shape of the middle CONV's window for the main path
filters -- python list of integers, defining the number of filters in the CONV layers of the main path
stage -- integer, used to name the layers, depending on their position in the network
block -- string/character, used to name the layers, depending on their position in the network
Returns:
X -- output of the identity block, tensor of shape (n_H, n_W, n_C)
"""
# defining name basis
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
# Retrieve Filters
F1, F2, F3 = filters
# Save the input value. You'll need this later to add back to the main path.
X_shortcut = X
# First component of main path
X = Conv3D(filters=F1,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='valid',
name=conv_name_base + '2a',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=4, name=bn_name_base + '2a')(X)
X = Activation('relu')(X)
### START CODE HERE ###
# Second component of main path (≈3 lines)
X = Conv3D(filters=F2,
kernel_size=(f, f, f),
strides=(1, 1, 1),
padding='same',
name=conv_name_base + '2b',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=4, name=bn_name_base + '2b')(X)
X = Activation('relu')(X)
# Third component of main path (≈2 lines)
X = Conv3D(filters=F3,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='valid',
name=conv_name_base + '2c',
kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=4, name=bn_name_base + '2c')(X)
# Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)
X = Add()([X, X_shortcut])
X = Activation('relu')(X)
### END CODE HERE ###
return X
# print("p[0].shape:", p[0].shape)
# print("p[1].shape:", p[1].shape)
# exit()
u_bias = Embedding(N, 1, embeddings_regularizer=l2(reg))(u) # (N, 1, 1)
m_bias = Embedding(M, 1, embeddings_regularizer=l2(reg))(m) # (N, 1, 1)
x = Dot(axes=2)([u_embedding, m_embedding]) # (N, 1, 1)
# submodel = Model([u, m], x)
# user_ids = df_train.userId.values[0:5]
# movie_ids = df_train.movie_idx.values[0:5]
# p = submodel.predict([user_ids, movie_ids])
# print("p.shape:", p.shape)
# exit()
x = Add()([x, u_bias, m_bias])
x = Flatten()(x) # (N, 1)
model = Model(inputs=[u, m], outputs=x)
model.compile(
loss='mse',
# optimizer='adam',
# optimizer=Adam(lr=0.01),
optimizer=SGD(lr=0.08, momentum=0.9),
metrics=['mse'],
)
r = model.fit(
x=[df_train.userId.values, df_train.movie_idx.values],
y=df_train.rating.values - mu,
epochs=epochs,
def standard_func_wbond(model, _emb_atoms, _emb_bonds, n_filters, adj, reg,
drop, step, prefix):
emb_shape = _emb_atoms.get_shape().as_list()
bond_shape = _emb_bonds.get_shape().as_list()
# print(emb_shape, bond_shape, adj.shape)
n_filters = n_filters
kernel_size_1, kernel_size_2 = (1, emb_shape[2]), (1, bond_shape[2] +
emb_shape[2])
strides_1, strides_2 = (1, emb_shape[2]), (1, bond_shape[2] + emb_shape[2])
kreg, breg = None, None
# if reg != 0:
# kreg, breg = regularizers.l2(reg), regularizers.l2(reg)
# else:
# kreg, breg = None, None
x1 = Conv2D(filters=n_filters,
kernel_size=kernel_size_1,
strides=strides_1,
padding='same',
data_format='channels_last',
activation=None,
kernel_regularizer=kreg,
bias_regularizer=breg,
name=str(step) + '_emb_conv')(
Lambda(lambda t: expand_last_dim(t))(_emb_atoms))
if reg != 0:
x1 = GaussianNoise(reg)(x1)
x1 = Lambda(lambda t: squeeze(t, -2))(x1)
# xbis = RepeatVector4D(emb_bonds.shape[1])(emb_atoms)
# xbis = Lambda(lambda t: dynamic_repeat4D(t),
# output_shape=(None, None, emb_shape[2]))(emb_atoms)
xbis = Lambda(lambda t: dynamic_repeataxis(t, 1, rep=None),
name=str(step) + 'x2repeat')(_emb_atoms)
# print("xbis", xbis)
x2 = Concatenate(axis=-1, name=str(step) + 'x2concat')([xbis, _emb_bonds])
# print("1 x2", x2)
x2bis = Lambda(lambda t: expand_last_dim(t),
name=str(step) + 'x2expand')(x2)
# print("1bis x2", x2bis)
x2_a = Conv2D(filters=n_filters,
kernel_size=kernel_size_2,
strides=strides_2,
padding='same',
data_format='channels_last',
activation=None,
kernel_regularizer=kreg,
bias_regularizer=breg,
name=str(step) + 'x2_emb_nei')(x2bis)
x2_b = Lambda(lambda t: squeeze(t, -2), name=str(step) + 'x2squeeze')(x2_a)
# print('2 x2', x2_b)
x2_c = Lambda(lambda args: K.batch_dot(args[0], args[1], axes=(2, 1)))(
[adj, x2_b])
# print('3 x2', x2_c)
# print("x1", x1)
if reg != 0:
x2_c = GaussianNoise(reg)(x2_c)
emb_atoms = Activation('relu')(Add()([x1, x2_c]))
if drop != 0:
emb_atoms = Dropout(drop, noise_shape=None, seed=None)(emb_atoms)
# print('emb_atoms', emb_atoms)
return emb_atoms
def build(num_users=7000,
num_movies=5000,
latent_dimension=120,
is_regularized=True,
is_biased=True,
**kwargs):
lamda = kwargs["lamda"] if "lamda" in kwargs else 1e-5
user_id_input = Input(shape=(1, ), name="UserID")
movie_id_input = Input(shape=(1, ), name="MovieID")
user_latent = Embedding(
num_users,
latent_dimension,
input_length=1,
embeddings_initializer="random_uniform",
embeddings_regularizer=l2(lamda) if is_regularized else None,
name="UserLatent")(user_id_input)
movie_latent = Embedding(
num_movies,
latent_dimension,
input_length=1,
embeddings_initializer="random_uniform",
embeddings_regularizer=l2(lamda) if is_regularized else None,
name="MovieLatent")(movie_id_input)
user_latent = Flatten(name="FlattenedUserLatent")(user_latent)
movie_latent = Flatten(name="FlattenedMovieLatent")(movie_latent)
output = Dot(1, name="NoBiasedRating")([user_latent, movie_latent])
model = Model([user_id_input, movie_id_input], output)
if is_biased:
bias_input = Input(shape=(1, ), name="GlobalBiasCoef")
bias = Embedding(1,
1,
input_length=1,
embeddings_initializer="random_uniform",
name="GlobalBias")(bias_input)
user_bias = Embedding(
num_users,
1,
input_length=1,
embeddings_initializer="random_uniform",
embeddings_regularizer=l2(lamda) if is_regularized else None,
name="UserBias")(user_id_input)
movie_bias = Embedding(
num_movies,
1,
input_length=1,
embeddings_initializer="random_uniform",
embeddings_regularizer=l2(lamda) if is_regularized else None,
name="MovieBias")(movie_id_input)
bias = Flatten(name="FlattenedGlobalBias")(bias)
user_bias = Flatten(name="FlattenedUserBias")(user_bias)
movie_bias = Flatten(name="FlattenedMovieBias")(movie_bias)
output = Add(name="Rating")([output, bias, user_bias, movie_bias])
model = Model([user_id_input, movie_id_input, bias_input], output)
model.compile(optimizer="adam", loss="mse", metrics=["mse"])
return model
def standard_func(model, _emb_atoms, n_filters, adj, reg, drop, step, prefix):
emb_shape = _emb_atoms.get_shape().as_list()
# print(emb_shape, adj.shape)
kernel_size, strides = (1, emb_shape[2]), (1, emb_shape[2])
kreg, breg = None, None
# if reg != 0:
# kreg, breg = regularizers.l2(reg), regularizers.l2(reg)
# else:
# kreg, breg = None, None
x1 = Conv2D(filters=n_filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
data_format='channels_last',
activation=None,
kernel_regularizer=kreg,
bias_regularizer=breg,
name=str(step) + prefix + '_emb_conv')(
Lambda(lambda t: expand_last_dim(t))(_emb_atoms))
if reg != 0:
x1 = GaussianNoise(reg)(x1)
x1 = Lambda(lambda t: squeeze(t, -2),
name=str(step) + prefix + "_squeeze_x1")(x1)
##### new
# print("1 x2", emb_atoms)
x2_a = Conv2D(filters=n_filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
data_format='channels_last',
activation=None,
kernel_regularizer=kreg,
bias_regularizer=breg,
name=str(step) + prefix + '_emb_nei')(
Lambda(lambda t: expand_last_dim(t))(_emb_atoms))
x2_b = Lambda(lambda t: squeeze(t, -2),
name=str(step) + prefix + "_squeeze_x2")(x2_a)
# print('2 x2', x2_b)
x2_c = Lambda(lambda arg: batch_dot_layer(arg[0], arg[1], axes=(2, 1)),
name=str(step) + prefix + "_emb_nei_sum")([adj, x2_b])
if reg != 0:
x2_c = GaussianNoise(reg)(x2_c)
# print('3 x2', x2_c)
# print("x1", x1)
##### old
# x2 = Lambda(lambda arg: batch_dot_layer(arg[0], arg[1], (2, 1)),
# name=str(step) + '_emb_nei_sum')([adj, emb_atoms])
# print('1 x2', x2)
# x2 = Conv2D(filters=n_filters, kernel_size=kernel_size, strides=strides,
# padding='same', data_format='channels_last',
# activation=None, kernel_regularizer=kreg, bias_regularizer=breg,
# name=str(step) + '_nei_conv')(Lambda(lambda t: expand_last_dim(t))(x2))
# print('2 x2', x2)
# x2 = Multiply()([norm, x2]) # , name=str(step) + '_emb_nei_sum_norm')
# print('3 x2', x2)
# x3 = x1 + x2 ; x3 = K.squeeze(ReLU()(x3), axis=-2) ; print('x3.shape', x3.shape)
emb_atoms = \
(Activation('relu', name=str(step) + prefix + '_emb_atoms')(Add(name="")([x1, x2_c])))
if drop != 0:
emb_atoms = Dropout(drop, noise_shape=None, seed=None)(emb_atoms)
else:
print('mol drop is 0 ###########')
return emb_atoms
def _main(args):
config_path = os.path.expanduser(args.config_path)
weights_path = os.path.expanduser(args.weights_path)
assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format(
config_path)
assert weights_path.endswith(
'.weights'), '{} is not a .weights file'.format(weights_path)
output_path = os.path.expanduser(args.output_path)
assert output_path.endswith(
'.h5'), 'output path {} is not a .h5 file'.format(output_path)
output_root = os.path.splitext(output_path)[0]
# Load weights and config.
print('Loading weights.')
weights_file = open(weights_path, 'rb')
major, minor, revision = np.ndarray(shape=(3, ),
dtype='int32',
buffer=weights_file.read(12))
if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000:
seen = np.ndarray(shape=(1, ),
dtype='int64',
buffer=weights_file.read(8))
else:
seen = np.ndarray(shape=(1, ),
dtype='int32',
buffer=weights_file.read(4))
print('Weights Header: ', major, minor, revision, seen)
print('Parsing Darknet config.')
unique_config_file = unique_config_sections(config_path)
cfg_parser = configparser.ConfigParser()
cfg_parser.read_file(unique_config_file)
print('Creating Keras model.')
input_layer = Input(shape=(None, None, 3))
prev_layer = input_layer
all_layers = []
weight_decay = float(cfg_parser['net_0']['decay']
) if 'net_0' in cfg_parser.sections() else 5e-4
count = 0
out_index = []
for section in cfg_parser.sections():
print('Parsing section {}'.format(section))
if section.startswith('convolutional'):
filters = int(cfg_parser[section]['filters'])
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
pad = int(cfg_parser[section]['pad'])
activation = cfg_parser[section]['activation']
batch_normalize = 'batch_normalize' in cfg_parser[section]
padding = 'same' if pad == 1 and stride == 1 else 'valid'
# Setting weights.
# Darknet serializes convolutional weights as:
# [bias/beta, [gamma, mean, variance], conv_weights]
prev_layer_shape = K.int_shape(prev_layer)
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
print('conv2d', 'bn' if batch_normalize else ' ', activation,
weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype='float32',
buffer=weights_file.read(filters * 4))
count += filters
if batch_normalize:
bn_weights = np.ndarray(shape=(3, filters),
dtype='float32',
buffer=weights_file.read(filters * 12))
count += 3 * filters
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2] # running var
]
conv_weights = np.ndarray(shape=darknet_w_shape,
dtype='float32',
buffer=weights_file.read(weights_size *
4))
count += weights_size
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
conv_weights = [conv_weights] if batch_normalize else [
conv_weights, conv_bias
]
# Handle activation.
act_fn = None
if activation == 'leaky':
pass # Add advanced activation later.
elif activation != 'linear':
raise ValueError(
'Unknown activation function `{}` in section {}'.format(
activation, section))
# Create Conv2D layer
if stride > 1:
# Darknet uses left and top padding instead of 'same' mode
prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer)
conv_layer = (Conv2D(filters, (size, size),
strides=(stride, stride),
kernel_regularizer=l2(weight_decay),
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
padding=padding))(prev_layer)
if batch_normalize:
conv_layer = (BatchNormalization(
weights=bn_weight_list))(conv_layer)
prev_layer = conv_layer
if activation == 'linear':
all_layers.append(prev_layer)
elif activation == 'leaky':
act_layer = LeakyReLU(alpha=0.1)(prev_layer)
prev_layer = act_layer
all_layers.append(act_layer)
elif section.startswith('route'):
ids = [int(i) for i in cfg_parser[section]['layers'].split(',')]
layers = [all_layers[i] for i in ids]
if len(layers) > 1:
print('Concatenating route layers:', layers)
concatenate_layer = Concatenate()(layers)
all_layers.append(concatenate_layer)
prev_layer = concatenate_layer
else:
skip_layer = layers[0] # only one layer to route
all_layers.append(skip_layer)
prev_layer = skip_layer
elif section.startswith('maxpool'):
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
all_layers.append(
MaxPooling2D(pool_size=(size, size),
strides=(stride, stride),
padding='same')(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('shortcut'):
index = int(cfg_parser[section]['from'])
activation = cfg_parser[section]['activation']
assert activation == 'linear', 'Only linear activation supported.'
all_layers.append(Add()([all_layers[index], prev_layer]))
prev_layer = all_layers[-1]
elif section.startswith('upsample'):
stride = int(cfg_parser[section]['stride'])
assert stride == 2, 'Only stride=2 supported.'
all_layers.append(UpSampling2D(stride)(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('yolo'):
out_index.append(len(all_layers) - 1)
all_layers.append(None)
prev_layer = all_layers[-1]
elif section.startswith('net'):
pass
else:
raise ValueError(
'Unsupported section header type: {}'.format(section))
# Create and save model.
if len(out_index) == 0:
out_index.append(len(all_layers) - 1)
model = Model(inputs=input_layer,
outputs=[all_layers[i] for i in out_index])
print(model.summary())
if args.weights_only:
model.save_weights('{}'.format(output_path))
print('Saved Keras weights to {}'.format(output_path))
else:
model.save('{}'.format(output_path))
print('Saved Keras model to {}'.format(output_path))
# Check to see if all weights have been read.
remaining_weights = len(weights_file.read()) / 4
weights_file.close()
print('Read {} of {} from Darknet weights.'.format(
count, count + remaining_weights))
if remaining_weights > 0:
print('Warning: {} unused weights'.format(remaining_weights))
if args.plot_model:
plot(model, to_file='{}.png'.format(output_root), show_shapes=True)
print('Saved model plot to {}.png'.format(output_root))
def network_builder(topology,
weight_decay,
weight_constraint,
dropout,
stage='block_optimization'):
if weight_decay is not None:
weight_decay = regularizers.l2(weight_decay)
if weight_constraint is not None:
weight_constraint = constraints.max_norm(weight_constraint, axis=0)
input_dim = topology[0]
output_dim = topology[-1][0]
nb_hidden_layer = len(topology) - 2
inputs = Input((input_dim, ), name='inputs')
hiddens = inputs
for layer_iter in range(nb_hidden_layer):
hidden_blocks = []
for block_iter in range(len(topology[1 + layer_iter])):
dense_output = Dense(topology[1 + layer_iter][block_iter],
kernel_initializer='he_normal',
kernel_regularizer=weight_decay,
kernel_constraint=weight_constraint,
name='dense%d_%d' %
(layer_iter, block_iter))(hiddens)
bn_output = BN(name='bn%d_%d' %
(layer_iter, block_iter))(dense_output)
act_output = Activation('relu')(bn_output)
if (stage=='block_optimization' and\
layer_iter == nb_hidden_layer -1 and\
block_iter == len(topology[1+layer_iter])-1 and\
dropout is not None) or\
(stage=='finetune' and dropout is not None):
dropout_output = Dropout(dropout)(act_output)
hidden_blocks.append(dropout_output)
else:
hidden_blocks.append(act_output)
# if last layer, attach output prediction block
if layer_iter == nb_hidden_layer - 1:
output_blocks = []
for block_idx, hidden_block in enumerate(hidden_blocks):
output_blocks.append(
Dense(output_dim,
kernel_initializer='he_normal',
kernel_regularizer=weight_decay,
kernel_constraint=weight_constraint,
name='output%d_%d' %
(nb_hidden_layer - 1, block_idx))(hidden_block))
# if not last hidden layer, concatenate all hidden blocks
else:
if len(hidden_blocks) > 1:
hiddens = Concatenate(axis=-1)(hidden_blocks)
else:
hiddens = hidden_blocks[0]
if len(output_blocks) > 1:
outputs = Add()(output_blocks)
else:
outputs = output_blocks[0]
output_activation = topology[-1][-1]
pre_predictions = Activation('linear', name='pre_predictions')(outputs)
if output_activation is not None:
predictions = Activation(output_activation,
name='predictions')(pre_predictions)
else:
predictions = Activation('linear', name='predictions')(pre_predictions)
model = Model(inputs=inputs, outputs=predictions)
return model
