def resnet_50_unet(input_shape):
resnet_base = resnet_50(input_shape=input_shape)
conv1 = resnet_base.get_layer("activation_1").output
conv2 = resnet_base.get_layer("activation_10").output
conv3 = resnet_base.get_layer("activation_22").output
conv4 = resnet_base.get_layer("activation_40").output
conv5 = resnet_base.get_layer("activation_49").output
up6 = concatenate([UpSampling2D()(conv5), conv4], axis=-1)
conv6 = conv_bn_relu(up6, 256, "conv6_1")
conv6 = conv_bn_relu(conv6, 256, "conv6_2")
up7 = concatenate([UpSampling2D()(conv6), conv3], axis=-1)
conv7 = conv_bn_relu(up7, 192, "conv7_1")
conv7 = conv_bn_relu(conv7, 192, "conv7_2")
up8 = concatenate([UpSampling2D()(conv7), conv2], axis=-1)
conv8 = conv_bn_relu(up8, 128, "conv8_1")
conv8 = conv_bn_relu(conv8, 128, "conv8_2")
up9 = concatenate([UpSampling2D()(conv8), conv1], axis=-1)
conv9 = conv_bn_relu(up9, 64, "conv9_1")
conv9 = conv_bn_relu(conv9, 64, "conv9_2")
up10 = concatenate([UpSampling2D()(conv9), resnet_base.input], axis=-1)
conv10 = conv_bn_relu(up10, 32, "conv10_1")
conv10 = conv_bn_relu(conv10, 32, "conv10_2")
x = SpatialDropout2D(0.5)(conv10)
x = Conv2D(1, (1, 1), activation="sigmoid")(x)
model = Model(resnet_base.input, x)
return model
def inception_unet(input_shape):
inception_base = InceptionV3Same(input_shape=input_shape)
conv1 = inception_base.get_layer("activation_3").output
conv2 = inception_base.get_layer("activation_5").output
conv3 = inception_base.get_layer("activation_29").output
conv4 = inception_base.get_layer("activation_75").output
conv5 = inception_base.get_layer("mixed10").output
up6 = concatenate([UpSampling2D()(conv5), conv4], axis=-1)
conv6 = conv_bn_relu(up6, 256, "conv6_1")
conv6 = conv_bn_relu(conv6, 256, "conv6_2")
up7 = concatenate([UpSampling2D()(conv6), conv3], axis=-1)
conv7 = conv_bn_relu(up7, 192, "conv7_1")
conv7 = conv_bn_relu(conv7, 192, "conv7_2")
up8 = concatenate([UpSampling2D()(conv7), conv2], axis=-1)
conv8 = conv_bn_relu(up8, 128, "conv8_1")
conv8 = conv_bn_relu(conv8, 128, "conv8_2")
up9 = concatenate([UpSampling2D()(conv8), conv1], axis=-1)
conv9 = conv_bn_relu(up9, 64, "conv9_1")
conv9 = conv_bn_relu(conv9, 64, "conv9_2")
up10 = UpSampling2D()(conv9)
conv10 = conv_bn_relu(up10, 32, "conv10_1")
conv10 = conv_bn_relu(conv10, 32, "conv10_2")
x = SpatialDropout2D(0.5)(conv10)
x = Conv2D(1, (1, 1), activation="sigmoid", name="mask")(x)
model = Model(inception_base.input, x)
return model
def basic_critic_model():
#track = Input(shape=INPUT_DIM[3]) Use either track or focus
#opponents = Input(shape=[INPUT_DIM[2]]) Most likely won't use as agents is racing by itself
actions = Input(shape=[ACTION_DIM])
focus = Input(shape=INPUT_DIM[0])
speed = Input(shape=INPUT_DIM[1]) # vector of speedX, speedY and speedZ
rpm = Input(shape=INPUT_DIM[2])
wheelSpinVel = Input(shape=INPUT_DIM[3])
focus = Input(shape=INPUT_DIM[0])
speed = Input(shape=INPUT_DIM[1]) # vector of speedX, speedY and speedZ
rpm = Input(shape=INPUT_DIM[2])
wheelSpinVel = Input(shape=INPUT_DIM[3])
speed_rpm = concatenate([speed, rpm])
speedh1 = Dense(150, activation='linear')(speed_rpm)
wheelSpinVelh1 = Dense(150, activation='linear')(wheelSpinVel)
combinedSpeed = add([speedh1, wheelSpinVelh1])
focush1 = Dense(150, activation='linear')(focus)
combined_layer = concatenate([focush1, combinedSpeed])
h3 = Dense(600, activation='relu')(combined_layer)
action_h = BatchNormalization()(Dense(600, activation='linear', init='glorot_normal')(actions))
combined = add([h3, action_h])
final = Dense(600, activation='relu')((combined))
Q = Dense(2, activation='linear', init='glorot_normal')((final))
model = Model(inputs=[focus, speed, rpm, wheelSpinVel, actions], outputs=[Q])
model.compile(loss='mse', optimizer=Adam(lr=1e-3))
print(model.summary())
return model
def basic_actor_model():
#track = Input(shape=INPUT_DIM[3]) Use either track or focus
#opponents = Input(shape=[INPUT_DIM[2]]) Most likely won't use as agents is racing by itself
focus = Input(shape=INPUT_DIM[0])
speed = Input(shape=INPUT_DIM[1]) # vector of speedX, speedY and speedZ
rpm = Input(shape=INPUT_DIM[2])
wheelSpinVel = Input(shape=INPUT_DIM[3])
speed_rpm = concatenate([speed, rpm])
speedh1 = Dense(150, activation='linear')(speed_rpm)
wheelSpinVelh1 = Dense(150, activation='linear')(wheelSpinVel)
combinedSpeed = add([speedh1, wheelSpinVelh1])
focush1 = Dense(150, activation='linear')(focus)
combined_layer = concatenate([focush1, combinedSpeed])
h3 = Dense(600, activation='relu')(combined_layer)
steering = Dense(1, activation='tanh', init='glorot_normal')(h3) #consider adding acceleration as an input to steering
acceleration = Dense(1, activation='sigmoid', init='glorot_normal')(h3)
output = concatenate([steering, acceleration])
model = Model(inputs=[focus, speed, rpm, wheelSpinVel], outputs=[output])
model.compile(loss='mse', optimizer=Adam(lr=1e-4))
print(model.summary())
return model
def network(n_lambda, depth, noise=0.0, activation='relu', n_filters=64, l2_reg=1e-7, n_mixture=8):
stokes_input = Input(shape=(n_lambda,4), name='stokes_input')
x = GaussianNoise(noise)(stokes_input)
x = Conv1D(n_filters, 3, activation=activation, padding='same', kernel_initializer='he_normal', name='conv_1', kernel_regularizer=l2(l2_reg))(x)
for i in range(depth):
x = residual(x, n_filters*(i+1), activation, l2_reg, strides=2)
intermediate = Flatten(name='flat')(x)
mu_input = Input(shape=(1,), name='mu_input')
intermediate_conv = concatenate([intermediate, mu_input], name='FC')
out_mu = Dense(n_mixture*9, name='FC_mu')(intermediate_conv)
out_sigma = Dense(n_mixture*9, activation=elu_modif, name='FC_sigma')(intermediate_conv)
out_alpha = Dense(n_mixture, activation='softmax', name='FC_alpha')(intermediate_conv)
out = concatenate([out_mu, out_sigma, out_alpha])
model = Model(inputs=[stokes_input, mu_input], outputs=out)
return model
def test_merge_concatenate():
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
o = layers.concatenate([i1, i2], axis=1)
assert o._keras_shape == (None, 8, 5)
model = models.Model([i1, i2], o)
concat_layer = layers.Concatenate(axis=1)
o2 = concat_layer([i1, i2])
assert concat_layer.output_shape == (None, 8, 5)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
assert out.shape == (2, 8, 5)
assert_allclose(out, np.concatenate([x1, x2], axis=1), atol=1e-4)
x3 = np.random.random((1, 1, 1))
nb_layers = 4
x_i = layers.Input(shape=(None, None))
x_list = [x_i]
x = x_i
for i in range(nb_layers):
x_list.append(x)
x = layers.concatenate(x_list, axis=1)
concat_model = models.Model(x_i, x)
concat_out = concat_model.predict([x3])
x3 = np.repeat(x3, 16, axis=1)
assert concat_out.shape == (1, 16, 1)
assert_allclose(concat_out, x3)
def AllDropOut(img_shape, out_ch=1, start_ch=64, activation='relu', dropout=0.1, batchnorm=False, residual=False):
i = Input(shape=img_shape)
# cb1 = conv_block(i, start_ch, activation, batchnorm, residual, dropout,stride=1)
# m1 = MaxPooling2D()(cb1)
# cb2 = conv_block(m1, int(start_ch*2), activation, batchnorm, residual, dropout,stride=1)
# m2 = MaxPooling2D()(cb2)
# cb3 = conv_block(m2, int(start_ch*4), activation, batchnorm, residual, dropout,stride=1)
# m3 = MaxPooling2D()(cb3)
# cb4 = conv_block(m3, int(start_ch*8), activation, batchnorm, residual, dropout,stride=1)
# m4 = MaxPooling2D()(cb4)
# cb5 = conv_block(m4, int(start_ch*16), activation, batchnorm, residual, dropout,stride=1)
# up1 = UpSampling2D()(cb5)
# up1 = concatenate([up1, cb4], axis=3)
# cb6 = conv_block(up1, int(start_ch*8), activation, batchnorm, residual, dropout,stride=1)
#
# up2 = UpSampling2D()(cb6)
# up2 = concatenate([up2, cb3], axis=3)
# cb7 = conv_block(up2, int(start_ch*4), activation, batchnorm, residual, dropout,stride=1)
#
# up3 = UpSampling2D()(cb7)
# up3 = concatenate([up3, cb2], axis=3)
# cb8 = conv_block(up3, int(start_ch*2), activation, batchnorm, residual, dropout,stride=1)
#
# up4 = UpSampling2D()(cb8)
# up4 = concatenate([up4, cb1], axis=3)
# cbLast = conv_block(up4, start_ch, activation, batchnorm, residual, dropout,stride=1)
cb1 = conv_block(i, 64, activation, False, residual, dropout,stride=1)#no batch norm at first layer
m1 = MaxPooling2D()(cb1)
cb2 = conv_block(m1, 64, activation, batchnorm, residual, dropout,stride=1)
m2 = MaxPooling2D()(cb2)
cb3 = conv_block(m2, 128, activation, batchnorm, residual, dropout,stride=1)
m3 = MaxPooling2D()(cb3)
cb4 = conv_block(m3, 128, activation, batchnorm, residual, dropout,stride=1)
m4 = MaxPooling2D()(cb4)
cb5 = conv_block(m4, 128, activation, batchnorm, residual, dropout,stride=1)
up1 = UpSampling2D()(cb5)
up1 = concatenate([up1, cb4], axis=3)
cb6 = conv_block(up1, 256, activation, batchnorm, residual, dropout,stride=1)
up2 = UpSampling2D()(cb6)
up2 = concatenate([up2, cb3], axis=3)
cb7 = conv_block(up2, 128, activation, batchnorm, residual, dropout,stride=1)
up3 = UpSampling2D()(cb7)
up3 = concatenate([up3, cb2], axis=3)
cb8 = conv_block(up3, 128, activation, batchnorm, residual, dropout,stride=1)
up4 = UpSampling2D()(cb8)
up4 = concatenate([up4, cb1], axis=3)
cbLast = conv_block(up4, 128, activation, batchnorm, residual, dropout,stride=1)
o = Conv2D(out_ch, 1, activation='sigmoid')(cbLast)
return Model(inputs=i, outputs=o)
def get_model(glove_embedding_lookup_table, fasttext_embedding_lookup_table, dropout, spatialDropout):
print('dropout : {0}'.format(dropout))
print('spatial dropout : {0}'.format(spatialDropout))
input_layer = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
## Glove embedding channel
glove_embedding_layer = Embedding(
input_dim=glove_embedding_lookup_table.shape[0],
output_dim=glove_embedding_lookup_table.shape[1],
weights=[glove_embedding_lookup_table],
trainable=False
)(input_layer)
glove_embedding_layer = SpatialDropout1D(spatialDropout)(glove_embedding_layer)
conv_1k = Conv1D(filters=128, kernel_size=1, padding='same', activation='relu')(glove_embedding_layer)
conv_2k = Conv1D(filters=128, kernel_size=2, padding='same', activation='relu')(glove_embedding_layer)
conv_3k = Conv1D(filters=128, kernel_size=3, padding='same', activation='relu')(glove_embedding_layer)
merge_1 = concatenate([conv_1k, conv_2k, conv_3k])
conv_1k = Conv1D(filters=64, kernel_size=1, padding='same', activation='relu')(merge_1)
conv_2k = Conv1D(filters=64, kernel_size=2, padding='same', activation='relu')(merge_1)
conv_3k = Conv1D(filters=64, kernel_size=3, padding='same', activation='relu')(merge_1)
conv_4k = Conv1D(filters=64, kernel_size=4, padding='same', activation='relu')(merge_1)
maxpool_1 = GlobalMaxPooling1D()(conv_1k)
maxpool_2 = GlobalMaxPooling1D()(conv_2k)
maxpool_3 = GlobalMaxPooling1D()(conv_3k)
maxpool_4 = GlobalMaxPooling1D()(conv_4k)
glove_multi_filters = [maxpool_1, maxpool_2, maxpool_3, maxpool_4]
## Fasttext embedding channel
fasttext_embedding_layer = Embedding(
input_dim=fasttext_embedding_lookup_table.shape[0],
output_dim=fasttext_embedding_lookup_table.shape[1],
weights=[fasttext_embedding_lookup_table],
trainable=False
)(input_layer)
fasttext_embedding_layer = SpatialDropout1D(spatialDropout)(fasttext_embedding_layer)
conv_1k = Conv1D(filters=128, kernel_size=1, padding='same', activation='tanh')(fasttext_embedding_layer)
conv_2k = Conv1D(filters=128, kernel_size=2, padding='same', activation='tanh')(fasttext_embedding_layer)
conv_3k = Conv1D(filters=128, kernel_size=3, padding='same', activation='tanh')(fasttext_embedding_layer)
merge_1 = concatenate([conv_1k, conv_2k, conv_3k])
conv_1k = Conv1D(filters=64, kernel_size=1, padding='same', activation='tanh')(merge_1)
conv_2k = Conv1D(filters=64, kernel_size=2, padding='same', activation='tanh')(merge_1)
conv_3k = Conv1D(filters=64, kernel_size=3, padding='same', activation='tanh')(merge_1)
conv_4k = Conv1D(filters=64, kernel_size=4, padding='same', activation='tanh')(merge_1)
maxpool_1 = GlobalMaxPooling1D()(conv_1k)
maxpool_2 = GlobalMaxPooling1D()(conv_2k)
maxpool_3 = GlobalMaxPooling1D()(conv_3k)
maxpool_4 = GlobalMaxPooling1D()(conv_4k)
fasttext_multi_filters = [maxpool_1, maxpool_2, maxpool_3, maxpool_4]
## Concatente
layer = concatenate(glove_multi_filters + fasttext_multi_filters)
layer = Dropout(dropout)(layer)
layer = Dense(400, kernel_initializer='he_uniform')(layer)
layer = PReLU()(layer)
layer = BatchNormalization()(layer)
layer = Dropout(dropout)(layer)
output_layer = Dense(6, activation='sigmoid')(layer)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='binary_crossentropy', optimizer=Nadam(), metrics=['acc'])
return model
def get_inception_v3_pre_post_model(img_dim, conv_dim, number_categories, model_weights = None):
popped, pre_model = get_inception_v3_pre_model(img_dim)
input_dims = (conv_dim, conv_dim, 2048)
# Take last 33 layers from inception_v3 with their starting weights!
mixed_9 = Input(shape = input_dims)
branch1x1 = popped[18](mixed_9)
branch1x1 = popped[12](branch1x1)
branch1x1 = popped[6](branch1x1)
branch3x3 = popped[29](mixed_9)
branch3x3 = popped[27](branch3x3)
branch3x3 = popped[25](branch3x3)
branch3x3_1 = popped[23](branch3x3)
branch3x3_1 = popped[17](branch3x3_1)
branch3x3_1 = popped[11](branch3x3_1)
branch3x3_2 = popped[22](branch3x3)
branch3x3_2 = popped[16](branch3x3_2)
branch3x3_2 = popped[10](branch3x3_2)
branch3x3 = concatenate([branch3x3_1, branch3x3_2], axis=3, name='mixed9_1')
branch3x3dbl = popped[32](mixed_9)
branch3x3dbl = popped[31](branch3x3dbl)
branch3x3dbl = popped[30](branch3x3dbl)
branch3x3dbl = popped[28](branch3x3dbl)
branch3x3dbl = popped[26](branch3x3dbl)
branch3x3dbl = popped[24](branch3x3dbl)
branch3x3dbl_1 = popped[21](branch3x3dbl)
branch3x3dbl_1 = popped[15](branch3x3dbl_1)
branch3x3dbl_1 = popped[9](branch3x3dbl_1)
branch3x3dbl_2 = popped[20](branch3x3dbl)
branch3x3dbl_2 = popped[14](branch3x3dbl_2)
branch3x3dbl_2 = popped[8](branch3x3dbl_2)
branch3x3dbl = concatenate([branch3x3dbl_1, branch3x3dbl_2], axis=3, name='concatenate_4')
branch_pool = popped[19](mixed_9)
branch_pool = popped[13](branch_pool)
branch_pool = popped[7](branch_pool)
branch_pool = popped[3](branch_pool)
x = concatenate([branch1x1, branch3x3, branch3x3dbl, branch_pool], axis=3, name='mixed10')
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(number_categories, activation='softmax', name='predictions')(x)
post_model = Model(mixed_9, x)
if model_weights is not None:
print('Loading model weights:', model_weights)
post_model.load_weights(model_weights)
return pre_model, post_model
def setup_model(self):
# TODO drop out?
inputs = Input((self.patch_size, self.patch_size, self.in_chan))
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(pool3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = Conv2D(512, (3, 3), activation='relu', padding='same')(pool4)
conv5 = Conv2D(512, (3, 3), activation='relu', padding='same')(conv5)
# up6 = merge([UpSampling2D(size=(2, 2))(conv5), conv4], mode='concat', concat_axis=1)
up6 = UpSampling2D(size=(2, 2))(conv5)
merge6 = concatenate([up6, conv4], axis=3)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(merge6)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv6)
# up7 = merge([UpSampling2D(size=(2, 2))(conv6), conv3], mode='concat', concat_axis=1)
up7 = UpSampling2D(size=(2, 2))(conv6)
merge7 = concatenate([up7, conv3], axis=3)
conv7 = Conv2D(128, (3, 3), activation='relu', padding='same')(merge7)
conv7 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv7)
#up8 = merge([UpSampling2D(size=(2, 2))(conv7), conv2], mode='concat', concat_axis=1)
up8 = UpSampling2D(size=(2, 2))(conv7)
merge8 = concatenate([up8, conv2], axis=3)
conv8 = Conv2D(64, (3, 3), activation='relu', padding='same')(merge8)
conv8 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv8)
#up9 = merge([UpSampling2D(size=(2, 2))(conv8), conv1], mode='concat', concat_axis=1)
up9 = UpSampling2D(size=(2, 2))(conv8)
merge9 = concatenate([up9, conv1], axis=3)
conv9 = Conv2D(32, (3, 3), activation='relu', padding='same')(merge9)
conv9 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv9)
conv10 = Conv2D(self.out_chan, (1, 1), activation='sigmoid')(conv9)
model = Model(inputs=inputs, outputs=conv10)
model.compile(optimizer=Adam(),
loss='binary_crossentropy',
metrics=[libs.metrics.jaccard_coef,
libs.metrics.jaccard_coef_int,
'accuracy'])
#print model.summary()
return model
def unet(vol_size, enc_nf, dec_nf, full_size=True):
"""
unet network for voxelmorph
Args:
vol_size: volume size. e.g. (256, 256, 256)
enc_nf: encoder filters. right now it needs to be to 1x4.
e.g. [16,32,32,32]
TODO: make this flexible.
dec_nf: encoder filters. right now it's forced to be 1x7.
e.g. [32,32,32,32,8,8,3]
TODO: make this flexible.
full_size
"""
# inputs
src = Input(shape=vol_size + (1,))
tgt = Input(shape=vol_size + (1,))
x_in = concatenate([src, tgt])
# down-sample path.
x0 = myConv(x_in, enc_nf[0], 2) # 80x96x112
x1 = myConv(x0, enc_nf[1], 2) # 40x48x56
x2 = myConv(x1, enc_nf[2], 2) # 20x24x28
x3 = myConv(x2, enc_nf[3], 2) # 10x12x14
# up-sample path.
x = myConv(x3, dec_nf[0])
x = UpSampling3D()(x)
x = concatenate([x, x2])
x = myConv(x, dec_nf[1])
x = UpSampling3D()(x)
x = concatenate([x, x1])
x = myConv(x, dec_nf[2])
x = UpSampling3D()(x)
x = concatenate([x, x0])
x = myConv(x, dec_nf[3])
x = myConv(x, dec_nf[4])
if full_size:
x = UpSampling3D()(x)
x = concatenate([x, x_in])
x = myConv(x, dec_nf[5])
# optional convolution
if (len(dec_nf) == 8):
x = myConv(x, dec_nf[6])
# transform the results into a flow.
flow = Conv3D(dec_nf[-1], kernel_size=3, padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow')(x)
# warp the source with the flow
y = Dense3DSpatialTransformer()([src, flow])
# prepare model
model = Model(inputs=[src, tgt], outputs=[y, flow])
return model
def get_model(self, input_shapes):
""" Main model function. """
feat_shape = input_shapes[0] # Features number
emb_shapes = input_shapes[1:] # Embeddings shapes
def build_mlp_model(input_shape, model_num=0, dense_layer_size=128, dropout_rate=0.9):
"""
The base MLP module. Dense + BatchNorm + Dropout.
Idea from Matei Ionita's kernel:
https://www.kaggle.com/mateiionita/taming-the-bert-a-baseline
"""
X_input = layers.Input([input_shape])
X = layers.Dense(dense_layer_size, name = 'mlp_dense_{}'.format(model_num), kernel_initializer=initializers.glorot_uniform(seed=self.seed))(X_input)
X = layers.BatchNormalization(name = 'mlp_bn_{}'.format(model_num))(X)
X = layers.Activation('relu')(X)
X = layers.Dropout(dropout_rate, seed = self.seed)(X)
model = models.Model(inputs = X_input, outputs = X, name = 'mlp_model_{}'.format(model_num))
return model
# Two models with inputs of shape (, 3*3*1014)
# from BERT Large cased and BERT Large uncased embeddings.
# Output shape is (, 112).
all_models = [build_mlp_model(emb_shape, model_num=i, dense_layer_size=112, dropout_rate=0.9) for i, emb_shape in enumerate(emb_shapes)]
# Two Siamese models with distances between
# Pronoun and A-term embeddings,
# Pronoun and B-term embeddings as inputs and shared weights.
# Input shape is (, 3*1024). Output shape is (, 2*112).
for i, emb_shape in enumerate(emb_shapes):
split_input = layers.Input([emb_shape])
split_model_shape = int(emb_shape / 3)
split_model = build_mlp_model(split_model_shape, model_num=len(all_models), dense_layer_size=112)
P = layers.Lambda(lambda x: x[:, :split_model_shape])(split_input)
A = layers.Lambda(lambda x: x[:, split_model_shape : split_model_shape*2])(split_input)
B = layers.Lambda(lambda x: x[:, split_model_shape*2 : split_model_shape*3])(split_input)
A_out = split_model(layers.Subtract()([P, A]))
B_out = split_model(layers.Subtract()([P, B]))
split_out = layers.concatenate([A_out, B_out], axis=-1)
merged_model = models.Model(inputs=split_input, outputs=split_out, name='split_model_{}'.format(i))
all_models.append(merged_model)
# One model
all_models.append(build_mlp_model(feat_shape, model_num=len(all_models), dense_layer_size=128, dropout_rate=0.8))
lambd = 0.02 # L2 regularization
# Combine all models into one model
# Concatenation of 5 models outputs
merged_out = layers.concatenate([model.output for model in all_models])
merged_out = layers.Dense(3, name = 'merged_output', kernel_regularizer = regularizers.l2(lambd), kernel_initializer=initializers.glorot_uniform(seed=self.seed))(merged_out)
merged_out = layers.BatchNormalization(name = 'merged_bn')(merged_out)
merged_out = layers.Activation('softmax')(merged_out)
# The final combined model.
combined_model = models.Model([model.input for model in all_models], outputs = merged_out, name = 'merged_model')
#print(combined_model.summary())
return combined_model
def create_model(model_input, word_embedding_matrix, maxlen=30, embeddings='glove', sent_embed='univ_sent', lr=1e-3):
# The visible layer
if embeddings != 'elmo':
in_dim, out_dim = word_embedding_matrix.shape
embedding_layer = Embedding(in_dim,
out_dim,
weights=[word_embedding_matrix],
input_length=maxlen,
trainable=False)
# Embedded version of the inputs
encoded_q1 = embedding_layer(model_input[0])
encoded_q2 = embedding_layer(model_input[1])
else:
encoded_q1 = Lambda(model_utils.ElmoEmbedding, output_shape=(maxlen, 1024))(model_input[0])
encoded_q2 = Lambda(model_utils.ElmoEmbedding, output_shape=(maxlen, 1024))(model_input[1])
q1_embed_sent = Lambda(model_utils.UniversalEmbedding, output_shape=(512,))(model_input[2])
q2_embed_sent = Lambda(model_utils.UniversalEmbedding, output_shape=(512,))(model_input[3])
sent1_dense = Dense(256, activation='relu')(q1_embed_sent)
sent2_dense = Dense(256, activation='relu')(q2_embed_sent)
distance_sent = Lambda(preprocessing.cosine_distance, output_shape=preprocessing.get_shape)(
[sent1_dense, sent2_dense])
shared_layer = Bidirectional(LSTM(n_hidden, kernel_initializer='glorot_uniform', return_sequences=True))
max_pool = GlobalMaxPooling1D()
output_q1 = shared_layer(encoded_q1)
output_q2 = shared_layer(encoded_q2)
output_q1 = max_pool(output_q1)
output_q2 = max_pool(output_q2)
# output_q1 = GaussianNoise(0.01)(output_q1)
# output_q2 = GaussianNoise(0.01)(output_q2)
abs_diff = Lambda(preprocessing.abs_difference, output_shape=preprocessing.get_shape)(
[output_q1, output_q2])
mult = Lambda(preprocessing.multiplication, output_shape=preprocessing.get_shape)([output_q1, output_q2])
if sent_embed == 'univ_sent':
merged = concatenate([output_q1, output_q2, abs_diff, mult, distance_sent])
else:
merged = concatenate([output_q1, output_q2, abs_diff, mult])
merged = Dense(n_hidden, activation='relu')(merged)
merged = Dropout(0.1)(merged)
merged = BatchNormalization()(merged)
output = Dense(1, activation='sigmoid', kernel_regularizer=regularizers.l2(0.0001),
bias_regularizer=regularizers.l2(0.0001))(merged)
# Pack it all up into a model
net = Model(model_input, [output])
net.compile(optimizer=SGD(lr=lr), loss='binary_crossentropy',
metrics=['binary_crossentropy', 'accuracy', model_utils.f1])
return net
def unet_other(weights=None, input_size=(512, 512, 4)):
inputs = Input(input_size)
conv1 = Conv2D(32, 3, activation='elu', padding='same', kernel_initializer='he_normal')(inputs)
conv1 = BatchNormalization()(conv1)
conv1 = Conv2D(32, 3, activation='elu', padding='same', kernel_initializer='he_normal')(conv1)
conv1 = BatchNormalization()(conv1)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(64, 3, activation='elu', padding='same', kernel_initializer='he_normal')(pool2)
conv2 = BatchNormalization()(conv2)
conv2 = Conv2D(64, 3, activation='elu', padding='same', kernel_initializer='he_normal')(conv2)
conv2 = BatchNormalization()(conv2)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(128, 3, activation='elu', padding='same', kernel_initializer='he_normal')(pool3)
conv3 = BatchNormalization()(conv3)
conv3 = Conv2D(128, 3, activation='elu', padding='same', kernel_initializer='he_normal')(conv3)
conv3 = BatchNormalization()(conv3)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(256, 3, activation='elu', padding='same', kernel_initializer='he_normal')(pool4)
conv4 = BatchNormalization()(conv4)
conv4 = Conv2D(256, 3, activation='elu', padding='same', kernel_initializer='he_normal')(conv4)
conv4 = BatchNormalization()(conv4)
up4 = UpSampling2D(size=(2,2))(conv4)
merge5 =concatenate([up4, conv3], axis=-1)
drop5 = Dropout(0.5)(merge5)
conv5 = Conv2D(128, 3, activation='elu', padding='same', kernel_initializer='he_normal')(drop5)
conv5 = Conv2D(128, 3, activation='elu', padding='same', kernel_initializer='he_normal')(conv5)
up5 = UpSampling2D(size=(2,2))(conv5)
merge6 = concatenate([up5, conv2], axis=-1)
drop6 = Dropout(0.5)(merge6)
conv6 = Conv2D(64, 3,activation='elu', padding='same', kernel_initializer='he_normal')(drop6)
conv6 = Conv2D(64, 3, activation='elu', padding='same', kernel_initializer='he_normal')(conv6)
up6 = UpSampling2D(size=(2,2))(conv6)
merge7 = concatenate([up6, conv1], axis=-1)
drop7 = Dropout(0.5)(merge7)
conv7 = Conv2D(32, 3, activation='elu',padding='same', kernel_initializer='he_normal')(drop7)
conv7 = Conv2D(32, 3, activation='elu', padding='same', kernel_initializer='he_normal')(conv7)
conv7 = Conv2D(1, 1, activation='sigmoid', padding='same', kernel_initializer='he_normal')(conv7)
model = Model(input=inputs, output=conv7)
model.compile(optimizer=Adam(lr=1e-4), loss=jaccard_coef_loss, metrics=['binary_crossentropy', jaccard_coef_int])
with open('model-summary-report.txt', 'w') as fh:
# Pass the file handle in as a lambda function to make it callable
model.summary(print_fn=lambda x: fh.write(x + '\n'))
if (weights):
model.load_weights(weights)
plot_model(model, to_file='unet-other-model.png',show_shapes=True) #vis model
return model
def build_reference_annotation_model(args):
'''Build Reference 1d CNN model for classifying variants with skip connected annotations.
Convolutions followed by dense connection, concatenated with annotations.
Dynamically sets input channels based on args via tensor_maps.get_tensor_channel_map_from_args(args)
Uses the functional API.
Prints out model summary.
Arguments
args.tensor_name: The name of the tensor mapping which data goes to which channels
args.annotation_set: The variant annotation set, perhaps from a HaplotypeCaller VCF.
args.labels: The output labels (e.g. SNP, NOT_SNP, INDEL, NOT_INDEL)
Returns
The keras model
'''
if K.image_data_format() == 'channels_last':
channel_axis = -1
else:
channel_axis = 1
channel_map = tensor_maps.get_tensor_channel_map_from_args(args)
reference = Input(shape=(args.window_size, len(channel_map)), name=args.tensor_name)
conv_width = 12
conv_dropout = 0.1
fc_dropout = 0.2
x = Conv1D(filters=256, kernel_size=conv_width, activation="relu", kernel_initializer='he_normal')(reference)
x = Conv1D(filters=256, kernel_size=conv_width, activation="relu", kernel_initializer='he_normal')(x)
x = Dropout(conv_dropout)(x)
x = Conv1D(filters=128, kernel_size=conv_width, activation="relu", kernel_initializer='he_normal')(x)
x = Dropout(conv_dropout)(x)
x = Flatten()(x)
annotations = Input(shape=(len(args.annotations),), name=args.annotation_set)
annos_normed = BatchNormalization(axis=channel_axis)(annotations)
annos_normed_x = Dense(units=40, kernel_initializer='normal', activation='relu')(annos_normed)
x = layers.concatenate([x, annos_normed_x], axis=channel_axis)
x = Dense(units=40, kernel_initializer='normal', activation='relu')(x)
x = Dropout(fc_dropout)(x)
x = layers.concatenate([x, annos_normed], axis=channel_axis)
prob_output = Dense(units=len(args.labels), kernel_initializer='glorot_normal', activation='softmax')(x)
model = Model(inputs=[reference, annotations], outputs=[prob_output])
adamo = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=1.)
model.compile(optimizer=adamo, loss='categorical_crossentropy', metrics=get_metrics(args.labels))
model.summary()
if os.path.exists(args.weights_hd5):
model.load_weights(args.weights_hd5, by_name=True)
print('Loaded model weights from:', args.weights_hd5)
return model
def build_estimator(self):
# Inputs
input_I = Input(shape=(self.n_lambda,1), name='stokes_I')
input_Q = Input(shape=(self.n_lambda,1), name='stokes_Q')
input_U = Input(shape=(self.n_lambda,1), name='stokes_U')
input_V = Input(shape=(self.n_lambda,1), name='stokes_V')
input_x = concatenate([input_I,input_Q,input_U,input_V])
y_true = Input(shape=(1,), name='y_true')
mu_input = Input(shape=(1,), name='mu_input')
# Neural network
x = Conv1D(64, 3, padding='same', kernel_initializer='he_normal', name='conv_1')(input_x)
x = PReLU()(x)
kernels = [64, 64, 64]
for i in range(3):
x = residual(x, kernels[i], 'prelu', strides=2)
intermediate = Flatten(name='flat')(x)
intermediate_conv = concatenate([intermediate, mu_input], name='FC')
weights = Dense(self.n_modes, activation='softmax', name='weights')(intermediate_conv)
if (self.infer_sigma == 'yes'):
sigma_global = Dense(1, activation='softplus', name='sigma')(intermediate_conv)
# Definition of the loss function
loss = Lambda(self.mdn_loss_function_infer, output_shape=(1,), name='loss')([y_true, weights, sigma_global])
else:
# Definition of the loss function
loss = Lambda(self.mdn_loss_function, output_shape=(1,), name='loss')([y_true, weights])
self.model = Model(inputs=[input_I,input_Q,input_U,input_V, y_true, mu_input], outputs=[loss])
# Compile with the loss weight set to None, so it will be omitted
self.model.load_weights("{0}_{1}_best.h5".format(self.root, self.var))
# Now generate a second network that ends up in the weights for later evaluation
if (self.infer_sigma == 'yes'):
self.model_weights = Model(inputs=self.model.input,
outputs=[self.model.get_layer('weights').output, self.model.get_layer('sigma').output])
else:
self.model_weights = Model(inputs=self.model.input,
outputs=self.model.get_layer('weights').output)
def test_merge_concatenate():
i1 = layers.Input(shape=(None, 5))
i2 = layers.Input(shape=(None, 5))
o = layers.concatenate([i1, i2], axis=1)
assert o._keras_shape == (None, None, 5)
model = models.Model([i1, i2], o)
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
o = layers.concatenate([i1, i2], axis=1)
assert o._keras_shape == (None, 8, 5)
model = models.Model([i1, i2], o)
concat_layer = layers.Concatenate(axis=1)
o2 = concat_layer([i1, i2])
assert concat_layer.output_shape == (None, 8, 5)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
assert out.shape == (2, 8, 5)
assert_allclose(out, np.concatenate([x1, x2], axis=1), atol=1e-4)
x3 = np.random.random((1, 1, 1))
nb_layers = 4
x_i = layers.Input(shape=(None, None))
x_list = [x_i]
x = x_i
for i in range(nb_layers):
x_list.append(x)
x = layers.concatenate(x_list, axis=1)
concat_model = models.Model(x_i, x)
concat_out = concat_model.predict([x3])
x3 = np.repeat(x3, 16, axis=1)
assert concat_out.shape == (1, 16, 1)
assert_allclose(concat_out, x3)
assert concat_layer.compute_mask([i1, i2], [None, None]) is None
assert np.all(K.eval(concat_layer.compute_mask(
[i1, i2], [K.variable(x1), K.variable(x2)])).reshape(-1))
# Test invalid use case
with pytest.raises(ValueError):
concat_layer.compute_mask([i1, i2], x1)
with pytest.raises(ValueError):
concat_layer.compute_mask(i1, [None, None])
with pytest.raises(ValueError):
concat_layer.compute_mask([i1, i2], [None])
with pytest.raises(ValueError):
concat_layer([i1])
def get_model(embedding_lookup_table, dropout, spatialDropout):
print('dropout : {0}'.format(dropout))
print('spatial dropout : {0}'.format(spatialDropout))
input_layer = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedding_layer = Embedding(
input_dim=embedding_lookup_table.shape[0],
output_dim=embedding_lookup_table.shape[1],
weights=[embedding_lookup_table],
trainable=False
)(input_layer)
embedding_layer = SpatialDropout1D(spatialDropout)(embedding_layer)
# First level conv
conv_1k = Conv1D(filters=150, kernel_size=1, padding='same', kernel_initializer='he_normal')(embedding_layer)
conv_2k = Conv1D(filters=150, kernel_size=2, padding='same', kernel_initializer='he_normal')(embedding_layer)
conv_3k = Conv1D(filters=150, kernel_size=3, padding='same', kernel_initializer='he_normal')(embedding_layer)
conv_1k = PReLU()(conv_1k)
conv_2k = PReLU()(conv_2k)
conv_3k = PReLU()(conv_3k)
merge_1 = concatenate([conv_1k, conv_2k, conv_3k])
# Second level conv and max pooling
conv_1k = Conv1D(filters=64, kernel_size=1, padding='same', kernel_initializer='he_normal')(merge_1)
conv_2k = Conv1D(filters=64, kernel_size=2, padding='same', kernel_initializer='he_normal')(merge_1)
conv_3k = Conv1D(filters=64, kernel_size=3, padding='same', kernel_initializer='he_normal')(merge_1)
conv_4k = Conv1D(filters=64, kernel_size=4, padding='same', kernel_initializer='he_normal')(merge_1)
conv_5k = Conv1D(filters=64, kernel_size=5, padding='same', kernel_initializer='he_normal')(merge_1)
conv_1k = PReLU()(conv_1k)
conv_2k = PReLU()(conv_2k)
conv_3k = PReLU()(conv_3k)
conv_4k = PReLU()(conv_4k)
conv_5k = PReLU()(conv_5k)
maxpool_1 = GlobalMaxPooling1D()(conv_1k)
maxpool_2 = GlobalMaxPooling1D()(conv_2k)
maxpool_3 = GlobalMaxPooling1D()(conv_3k)
maxpool_4 = GlobalMaxPooling1D()(conv_4k)
maxpool_5 = GlobalMaxPooling1D()(conv_5k)
layer = concatenate([maxpool_1, maxpool_2, maxpool_3, maxpool_4, maxpool_5])
layer = Dropout(dropout)(layer)
layer = Dense(400, kernel_initializer='he_normal')(layer)
layer = PReLU()(layer)
layer = BatchNormalization()(layer)
layer = Dropout(dropout)(layer)
output_layer = Dense(6, activation='sigmoid')(layer)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='binary_crossentropy', optimizer=Nadam(), metrics=['acc'])
return model
def get_model(embedding_lookup_table, dropout, spatial_dropout):
input_layer = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedding_layer = Embedding(
input_dim=embedding_lookup_table.shape[0],
output_dim=embedding_lookup_table.shape[1],
weights=[embedding_lookup_table],
# trainable=False
)(input_layer)
layer = embedding_layer
maxpool_embed = GlobalMaxPooling1D()(layer)
# spatial_dropout = spatial_dropout - 0.002 + np.random.rand() * 0.004
embed_after_sdp = SpatialDropout1D(spatial_dropout)(layer)
print('spatial dropout : {0}'.format(spatial_dropout))
gru_one = Bidirectional(CuDNNGRU(units=64, return_sequences=True))(embed_after_sdp)
# hyper-parameter vibration
dropout = dropout - 0.002 + np.random.rand() * 0.004
print('dropout : {0}'.format(dropout))
gru_one_after_dp = Dropout(dropout)(gru_one)
gru_two = Bidirectional(CuDNNGRU(units=64, return_sequences=True))(gru_one_after_dp)
layer = concatenate([gru_two, gru_one, maxpool_embed])
layer = AttentionWeightedAverage()(layer)
output_layer = Dense(6, activation='sigmoid')(layer)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='binary_crossentropy', optimizer=Nadam(), metrics=['acc'])
return model
def test_layer_sharing_at_heterogeneous_depth_with_concat():
input_shape = (16, 9, 3)
input_layer = Input(shape=input_shape)
A = Dense(3, name='dense_A')
B = Dense(3, name='dense_B')
C = Dense(3, name='dense_C')
x1 = B(A(input_layer))
x2 = A(C(input_layer))
output = layers.concatenate([x1, x2])
M = Model(inputs=input_layer, outputs=output)
x_val = np.random.random((10, 16, 9, 3))
output_val = M.predict(x_val)
config = M.get_config()
weights = M.get_weights()
M2 = Model.from_config(config)
M2.set_weights(weights)
output_val_2 = M2.predict(x_val)
np.testing.assert_allclose(output_val, output_val_2, atol=1e-6)
def network_dropout(n_lambda, n_classes, depth, noise=0.0, activation='relu', n_filters=64, l2_reg=1e-7, wd=0.0, dd=0.0):
stokes_input = Input(shape=(n_lambda,4), name='stokes_input')
x = GaussianNoise(noise)(stokes_input)
x = ConcreteDropout(Conv1D(n_filters, 3, activation=activation, padding='same', kernel_initializer='he_normal', name='conv_1',
kernel_regularizer=l2(l2_reg)),weight_regularizer=wd, dropout_regularizer=dd)(x)
for i in range(depth):
x = residual_dropout(x, n_filters*(i+1), activation, l2_reg, 2, wd, dd)
intermediate = Flatten(name='flat')(x)
mu_input = Input(shape=(1,), name='mu_input')
intermediate_conv = concatenate([intermediate, mu_input], name='FC')
x = ConcreteDropout(Dense(3*n_classes, activation='relu', name='FC_tau'),weight_regularizer=wd, dropout_regularizer=dd)(intermediate_conv)
out_tau = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_tau'),weight_regularizer=wd, dropout_regularizer=dd)(x)
out_v = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_v'),weight_regularizer=wd, dropout_regularizer=dd)(x)
out_vth = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_vth'),weight_regularizer=wd, dropout_regularizer=dd)(x)
out_a = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_a'),weight_regularizer=wd, dropout_regularizer=dd)(x)
out_B = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_B'),weight_regularizer=wd, dropout_regularizer=dd)(x)
out_thB = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_thB'),weight_regularizer=wd, dropout_regularizer=dd)(x)
out_phiB = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_phiB'),weight_regularizer=wd, dropout_regularizer=dd)(x)
out_ThB = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_ThB'),weight_regularizer=wd, dropout_regularizer=dd)(x)
out_PhiB = ConcreteDropout(Dense(n_classes, activation='softmax', name='out_PhiB'),weight_regularizer=wd, dropout_regularizer=dd)(x)
model = Model(inputs=[stokes_input, mu_input], outputs=[out_tau, out_v, out_vth, out_a, out_B, out_thB, out_phiB, out_ThB, out_PhiB])
return model
def network_nodropout(n_lambda, n_classes, depth, noise=0.0, activation='relu', n_filters=64, l2_reg=1e-7):
stokes_input = Input(shape=(n_lambda,1), name='stokes_input')
x = GaussianNoise(noise)(stokes_input)
x = Conv1D(n_filters, 3, activation=activation, padding='same', kernel_initializer='he_normal', name='conv_1',
kernel_regularizer=l2(l2_reg))(x)
for i in range(depth):
x = residual(x, n_filters*(i+1), activation, l2_reg, 1)
intermediate = GlobalAveragePooling1D(name='flat')(x)
mu_input = Input(shape=(1,), name='mu_input')
x = concatenate([intermediate, mu_input], name='FC')
out_tau = Dense(n_classes, activation='softmax', name='out_tau')(x)
out_v = Dense(n_classes, activation='softmax', name='out_v')(x)
out_vth = Dense(n_classes, activation='softmax', name='out_vth')(x)
out_a = Dense(n_classes, activation='softmax', name='out_a')(x)
# out_B = Dense(n_classes, activation='softmax', name='out_B')(x)
# out_thB = Dense(n_classes, activation='softmax', name='out_thB')(x)
# out_phiB = Dense(n_classes, activation='softmax', name='out_phiB')(x)
# out_ThB = Dense(n_classes, activation='softmax', name='out_ThB')(x)
# out_PhiB = Dense(n_classes, activation='softmax', name='out_PhiB')(x)
# model = Model(inputs=[stokes_input, mu_input], outputs=[out_tau, out_v, out_vth, out_a, out_B, out_thB, out_phiB, out_ThB, out_PhiB])
model = Model(inputs=[stokes_input, mu_input], outputs=[out_tau, out_v, out_vth, out_a])
return model
def inception_block_1c(X):
X_3x3 = fr_utils.conv2d_bn(X,
layer='inception_3c_3x3',
cv1_out=128,
cv1_filter=(1, 1),
cv2_out=256,
cv2_filter=(3, 3),
cv2_strides=(2, 2),
padding=(1, 1))
X_5x5 = fr_utils.conv2d_bn(X,
layer='inception_3c_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(2, 2),
padding=(2, 2))
X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X)
X_pool = ZeroPadding2D(padding=((0, 1), (0, 1)), data_format='channels_first')(X_pool)
inception = concatenate([X_3x3, X_5x5, X_pool], axis=1)
return inception
def L3_merge_audio_vision_models(vision_model, x_i, audio_model, x_a, model_name, layer_size=128):
"""
Merges the audio and vision subnetworks and adds additional fully connected
layers in the fashion of the model used in Look, Listen and Learn
Relja Arandjelovic and (2017). Look, Listen and Learn. CoRR, abs/1705.08168, .
Returns
-------
model: L3 CNN model
(Type: keras.models.Model)
inputs: Model inputs
(Type: list[keras.layers.Input])
outputs: Model outputs
(Type: keras.layers.Layer)
"""
# Merge the subnetworks
weight_decay = 1e-5
y = concatenate([vision_model(x_i), audio_model(x_a)])
y = Dense(layer_size, activation='relu',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(weight_decay))(y)
y = Dense(2, activation='softmax',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(weight_decay))(y)
m = Model(inputs=[x_i, x_a], outputs=y)
m.name = model_name
return m, [x_i, x_a], y
def build_model(lr = 0.0, lr_d = 0.0, units = 0, dr = 0.0):
inp = Input(shape = (max_len,))
x = Embedding(max_features, embed_size, weights = [embedding_matrix], trainable = False)(inp)
x1 = SpatialDropout1D(dr)(x)
x = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)
x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform")(x)
y = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x1)
y = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform")(y)
avg_pool1 = GlobalAveragePooling1D()(x)
max_pool1 = GlobalMaxPooling1D()(x)
avg_pool2 = GlobalAveragePooling1D()(y)
max_pool2 = GlobalMaxPooling1D()(y)
x = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
x = Dense(6, activation = "sigmoid")(x)
model = Model(inputs = inp, outputs = x)
model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
history = model.fit(X_train, Y_train, batch_size = 128, epochs = 3, validation_data = (X_valid, Y_valid),
verbose = 1, callbacks = [ra_val, check_point, early_stop])
model = load_model(file_path)
return model
def inception_block_1b(X):
X_3x3 = Conv2D(96, (1, 1), data_format='channels_first', name='inception_3b_3x3_conv1')(X)
X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_3x3_bn1')(X_3x3)
X_3x3 = Activation('relu')(X_3x3)
X_3x3 = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_3x3)
X_3x3 = Conv2D(128, (3, 3), data_format='channels_first', name='inception_3b_3x3_conv2')(X_3x3)
X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_3x3_bn2')(X_3x3)
X_3x3 = Activation('relu')(X_3x3)
X_5x5 = Conv2D(32, (1, 1), data_format='channels_first', name='inception_3b_5x5_conv1')(X)
X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_5x5_bn1')(X_5x5)
X_5x5 = Activation('relu')(X_5x5)
X_5x5 = ZeroPadding2D(padding=(2, 2), data_format='channels_first')(X_5x5)
X_5x5 = Conv2D(64, (5, 5), data_format='channels_first', name='inception_3b_5x5_conv2')(X_5x5)
X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_5x5_bn2')(X_5x5)
X_5x5 = Activation('relu')(X_5x5)
X_pool = AveragePooling2D(pool_size=(3, 3), strides=(3, 3), data_format='channels_first')(X)
X_pool = Conv2D(64, (1, 1), data_format='channels_first', name='inception_3b_pool_conv')(X_pool)
X_pool = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_pool_bn')(X_pool)
X_pool = Activation('relu')(X_pool)
X_pool = ZeroPadding2D(padding=(4, 4), data_format='channels_first')(X_pool)
X_1x1 = Conv2D(64, (1, 1), data_format='channels_first', name='inception_3b_1x1_conv')(X)
X_1x1 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3b_1x1_bn')(X_1x1)
X_1x1 = Activation('relu')(X_1x1)
inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1)
return inception
def inception_block_1a(X):
"""
Implementation of an inception block
"""
X_3x3 = Conv2D(96, (1, 1), data_format='channels_first', name ='inception_3a_3x3_conv1')(X)
X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name = 'inception_3a_3x3_bn1')(X_3x3)
X_3x3 = Activation('relu')(X_3x3)
X_3x3 = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_3x3)
X_3x3 = Conv2D(128, (3, 3), data_format='channels_first', name='inception_3a_3x3_conv2')(X_3x3)
X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_3x3_bn2')(X_3x3)
X_3x3 = Activation('relu')(X_3x3)
X_5x5 = Conv2D(16, (1, 1), data_format='channels_first', name='inception_3a_5x5_conv1')(X)
X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_5x5_bn1')(X_5x5)
X_5x5 = Activation('relu')(X_5x5)
X_5x5 = ZeroPadding2D(padding=(2, 2), data_format='channels_first')(X_5x5)
X_5x5 = Conv2D(32, (5, 5), data_format='channels_first', name='inception_3a_5x5_conv2')(X_5x5)
X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_5x5_bn2')(X_5x5)
X_5x5 = Activation('relu')(X_5x5)
X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X)
X_pool = Conv2D(32, (1, 1), data_format='channels_first', name='inception_3a_pool_conv')(X_pool)
X_pool = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_pool_bn')(X_pool)
X_pool = Activation('relu')(X_pool)
X_pool = ZeroPadding2D(padding=((3, 4), (3, 4)), data_format='channels_first')(X_pool)
X_1x1 = Conv2D(64, (1, 1), data_format='channels_first', name='inception_3a_1x1_conv')(X)
X_1x1 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_1x1_bn')(X_1x1)
X_1x1 = Activation('relu')(X_1x1)
# CONCAT
inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1)
return inception
def inception_block_2a(X):
X_3x3 = fr_utils.conv2d_bn(X,
layer='inception_4a_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=192,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
X_5x5 = fr_utils.conv2d_bn(X,
layer='inception_4a_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(1, 1),
padding=(2, 2))
X_pool = AveragePooling2D(pool_size=(3, 3), strides=(3, 3), data_format='channels_first')(X)
X_pool = fr_utils.conv2d_bn(X_pool,
layer='inception_4a_pool',
cv1_out=128,
cv1_filter=(1, 1),
padding=(2, 2))
X_1x1 = fr_utils.conv2d_bn(X,
layer='inception_4a_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1)
return inception
def build(self):
input_encoded_m = self.encoders_m(self.input_sequence)
input_encoded_c = self.encoders_c(self.input_sequence)
question_encoded = self.encoders_question(self.question)
match = dot([input_encoded_m, question_encoded], axes=(2, 2))
match = Activation('softmax')(match)
# add the match matrix with the second input vector sequence
response = add([match, input_encoded_c]) # (samples, story_maxlen, query_maxlen)
response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen)
# concatenate the match matrix with the question vector sequence
answer = concatenate([response, question_encoded])
# the original paper uses a matrix multiplication for this reduction step.
# we choose to use a RNN instead.
answer = LSTM(32)(answer) # (samples, 32)
# one regularization layer -- more would probably be needed.
answer = Dropout(0.3)(answer)
answer = Dense(self.vocab_size)(answer) # (samples, vocab_size)
# we output a probability distribution over the vocabulary
answer = Activation('softmax')(answer)
# build the final model
model = Model([self.input_sequence, self.question], answer)
self.model = model
return model
def build_estimator(self):
# Inputs
input_x = Input(shape=(self.n_lambda,4), name='stokes_input')
y_true = Input(shape=(1,), name='y_true')
mu_input = Input(shape=(1,), name='mu_input')
# Neural network
x = Conv1D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal', name='conv_1')(input_x)
for i in range(3):
x = residual(x, 64*(i+1), 'relu', strides=2)
intermediate = Flatten(name='flat')(x)
intermediate_conv = concatenate([intermediate, mu_input], name='FC')
# Output weights
weights = Dense(self.n_modes, activation='softmax', name='weights')(intermediate_conv)
# Definition of the loss function
loss = Lambda(self.mdn_loss_function, output_shape=(1,), name='loss')([y_true, weights])
self.model = Model(inputs=[input_x, y_true, mu_input], outputs=[loss])
#self.model.add_loss(loss)
# Compile with the loss weight set to None, so it will be omitted
#self.model.compile(loss=[None], loss_weights=[None], optimizer=Adam(lr=self.lr))
self.model.load_weights("{0}_{1}_best.h5".format(self.root, self.var))
# Now generate a second network that ends up in the weights for later evaluation
self.model_weights = Model(inputs=self.model.input,
outputs=self.model.get_layer('weights').output)
def InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format="channels_last"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
Note that the default input image size for this model is 299x299.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or "imagenet" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=299,
min_size=139,
data_format=K.image_data_format(),
include_top=include_top)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
img_input = Input(tensor=input_tensor, shape=input_shape)
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid')
x = conv2d_bn(x, 32, 3, 3, padding='valid')
x = conv2d_bn(x, 64, 3, 3)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv2d_bn(x, 80, 1, 1, padding='valid')
x = conv2d_bn(x, 192, 3, 3, padding='valid')
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl,
96,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3,
320,
3,
3,
strides=(2, 2),
padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(branch7x7x3,
192,
3,
3,
strides=(2, 2),
padding='valid')
branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate([branch3x3, branch7x7x3, branch_pool],
axis=channel_axis,
name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate([branch3x3_1, branch3x3_2],
axis=channel_axis,
name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate([branch3x3dbl_1, branch3x3dbl_2],
axis=channel_axis)
branch_pool = AveragePooling2D((3, 3), strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
if include_top:
# Classification block
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='inception_v3')
# load weights
if weights == 'imagenet':
if K.image_data_format() == 'channels_first':
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
if include_top:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='9a0d58056eeedaa3f26cb7ebd46da564')
else:
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='bcbd6486424b2319ff4ef7d526e38f63')
model.load_weights(weights_path)
if K.backend() == 'theano':
convert_all_kernels_in_model(model)
return model
def build_resnet(Traniable_embedding, embedding_matrix, max_len, kmer_size,
metrics, classes_1, classes_2, classes_3, classes_4,
classes_5, classes_6):
inp = Input(shape=(max_len, ), dtype='uint16')
max_features = 4**kmer_size + 1
if Traniable_embedding == True:
main = Embedding(5, 128)(inp)
else:
#main = Embedding(max_features, vector_size, weights=[embedding_matrix],trainable=False)(inp)
main = embedding_matrix(inp)
main = Conv1D(filters=64, kernel_size=3, padding='same')(main)
i_l1 = MaxPooling1D(pool_size=2)(main)
main = Conv1D(filters=64, kernel_size=3, padding='same')(main)
main = BatchNormalization()(main)
main = Activation('relu')(main)
main = Conv1D(filters=64, kernel_size=3, padding='same')(main)
main = concatenate([main, i_l1], axis=1)
main = BatchNormalization()(main)
main = Activation('relu')(main)
main = MaxPooling1D(pool_size=2)(main)
i_l1 = Conv1D(filters=128, kernel_size=1, padding='same')(main)
main = Conv1D(filters=128, kernel_size=3, padding='same')(main)
main = BatchNormalization()(main)
main = Activation('relu')(main)
main = Conv1D(filters=128, kernel_size=3, padding='same')(main)
main = concatenate([main, i_l1], axis=1)
main = BatchNormalization()(main)
main = Activation('relu')(main)
main = MaxPooling1D(pool_size=2)(main)
i_l1 = Conv1D(filters=256, kernel_size=1, padding='same')(main)
main = Conv1D(filters=256, kernel_size=3, padding='same')(main)
main = BatchNormalization()(main)
main = Activation('relu')(main)
main = Conv1D(filters=256, kernel_size=3, padding='same')(main)
main = concatenate([main, i_l1], axis=1)
main = BatchNormalization()(main)
main = Activation('relu')(main)
main = MaxPooling1D(pool_size=2)(main)
i_l1 = Conv1D(filters=512, kernel_size=1, padding='same')(main)
main = Conv1D(filters=512, kernel_size=3, padding='same')(main)
main = BatchNormalization()(main)
main = Activation('relu')(main)
main = Conv1D(filters=512, kernel_size=3, padding='same')(main)
main = concatenate([main, i_l1], axis=1)
main = BatchNormalization()(main)
main = Activation('relu')(main)
main = MaxPooling1D(pool_size=2)(main)
main = GlobalMaxPool1D()(main)
main = Dense(1024, activation='relu')(main) # Should be 4096
main = Dense(512, activation='relu')(main) # Should be 2048
out1 = Dense(classes_1, activation='softmax')(main)
out2 = Dense(classes_2, activation='softmax')(main)
out3 = Dense(classes_3, activation='softmax')(main)
out4 = Dense(classes_4, activation='softmax')(main)
out5 = Dense(classes_5, activation='softmax')(main)
if classes_6 != 0:
out6 = Dense(classes_6, activation='softmax')(main)
else:
pass
if metrics == 'f1':
metrics = f1
elif metrics == 'precision-recall':
metrics = [keras_metrics.precision(), keras_metrics.recall()]
else:
pass
model = Model(inputs=[inp], outputs=[out1, out2, out3, out4, out5])
optimizer = AdamW(lr=0.001,
beta_1=0.9,
beta_2=0.999,
weight_decay=1e-4,
epsilon=1e-8,
decay=0.)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizer,
metrics=[metrics])
return model
def build(width, height, depth, classes, pooling_regions = [1, 3], weights="imagenet"):
assert(isinstance(classes, list), 'Must be list type.')
assert(len(classes) == 3, 'Must be 3 elements in the list.')
inpt = Input(shape=(width, height, depth))
#padding = 'same'，填充为(步长-1）/2,还可以用ZeroPadding2D((3,3))
x = hiarBayesGoogLeNet.Conv2d_BN(inpt, 64, (7,7), strides=(2,2), padding='same', name="conv1_7x7_s2")
x = MaxPooling2D(pool_size=(3,3), strides=(2,2), padding='same')(x)
x = hiarBayesGoogLeNet.Conv2d_BN(x, 192, (3,3), strides=(1,1), padding='same', name="conv2_3x3")
x = MaxPooling2D(pool_size=(3,3), strides=(2,2), padding='same')(x)
"""
x = hiarBayesGoogLeNet.Inception(x, 64, name="inception_3a")#256
x = hiarBayesGoogLeNet.Inception(x, 120, name="inception_3b")#480
"""
x = hiarBayesGoogLeNet.Inception(x, [64,96,128,16,32,32], name="inception_3a")#256
x = hiarBayesGoogLeNet.Inception(x, [128,128,192,32,96,64], name="inception_3b")#480
x = MaxPooling2D(pool_size=(3,3), strides=(2,2), padding='same')(x)
"""
x = hiarBayesGoogLeNet.Inception(x, 128, name="inception_4a")#512
x = hiarBayesGoogLeNet.Inception(x, 128, name="inception_4b")
x = hiarBayesGoogLeNet.Inception(x, 128, name="inception_4c")
x = hiarBayesGoogLeNet.Inception(x, 132, name="inception_4d")#528
x = hiarBayesGoogLeNet.Inception(x, 208, name="inception_4e")#832
"""
x = hiarBayesGoogLeNet.Inception(x, [192,96,208,16,48,64], name="inception_4a")#512
fea_low = x
#fea_low = Conv2D(512, (3, 3), padding='same', activation='relu', name='conv1_e')(x)
#fea_low = GlobalAveragePooling2D()(x)#, name="gap_low"
#fea_low = Dense(512, activation='relu')(fea_low)
x = hiarBayesGoogLeNet.Inception(x, [160,112,224,24,64,64], name="inception_4b")
x = hiarBayesGoogLeNet.Inception(x, [128,128,256,24,64,64], name="inception_4c")
x = hiarBayesGoogLeNet.Inception(x, [112,144,288,32,64,64], name="inception_4d")#528
fea_mid = x
#fea_mid = Conv2D(512, (3, 3), padding='same', activation='relu', name='conv2_e')(x)
#fea_mid = GlobalAveragePooling2D()(x)#, name="gap_mid"
#fea_mid = Dense(512, activation='relu')(fea_mid)
x = hiarBayesGoogLeNet.Inception(x, [256,160,320,32,128,128], name="inception_4e")#832
x = MaxPooling2D(pool_size=(3,3), strides=(2,2), padding='same')(x)
"""
x = hiarBayesGoogLeNet.Inception(x, 208, name="inception_5a")
x = hiarBayesGoogLeNet.Inception(x, 256, name="inception_5b")#1024
"""
x = hiarBayesGoogLeNet.Inception(x, [256,160,320,32,128,128], name="inception_5a")
x = hiarBayesGoogLeNet.Inception(x, [384,192,384,48,128,128], name="inception_5b")#1024
fea_hig = x
#fea_hig = Conv2D(1024, (3, 3), padding='same', activation='relu', name='conv3_e')(x)
#fea_hig = GlobalAveragePooling2D()(x)#, name="gap_hig"
#fea_hig = Dense(1024, activation='relu')(fea_hig)
"""
predictions_low = Dense(classes[0], name="low", activation="sigmoid")(fea_low)#
predictions_mid_hs = Dense(classes[1], name="middle_hs", activation="sigmoid")(fea_mid)#
predictions_mid_ub = Dense(classes[2], name="middle_ub", activation="sigmoid")(fea_mid)#
predictions_mid_lb = Dense(classes[3], name="middle_lb", activation="sigmoid")(fea_mid)#
predictions_mid_sh = Dense(classes[4], name="middle_sh", activation="sigmoid")(fea_mid)#
predictions_mid_at = Dense(classes[5], name="middle_at", activation="sigmoid")(fea_mid)#
predictions_mid_ot = Dense(classes[6], name="middle_ot", activation="sigmoid")(fea_mid)#
predictions_hig = Dense(classes[7], name="high_fea", activation="sigmoid")(fea_hig)#
"""
fea_low = Conv2D(512, (3, 3), padding='same', activation='relu')(fea_low)
#fea_low = Flatten()(fea_low)
#fea_low = Dense(512, activation='relu')(fea_low)
fea_low = GlobalAveragePooling2D()(fea_low)
predictions_low = Dense(classes[0], name="low", activation="sigmoid")(fea_low)#
fea_mid_hs = Conv2D(512, (3, 3), padding='same', activation='relu')(fea_mid)
#fea_mid_hs = Flatten()(fea_mid_hs)
#fea_mid_hs = Dense(512, activation='relu')(fea_mid_hs)
fea_mid_hs = GlobalAveragePooling2D()(fea_mid_hs)
predictions_mid_hs = Dense(classes[1], name="middle_hs", activation="sigmoid")(fea_mid_hs)#
fea_mid_ub = Conv2D(512, (3, 3), padding='same', activation='relu')(fea_mid)
#fea_mid_ub = Flatten()(fea_mid_ub)
#fea_mid_ub = Dense(512, activation='relu')(fea_mid_ub)
fea_mid_ub = GlobalAveragePooling2D()(fea_mid_ub)
predictions_mid_ub = Dense(classes[2], name="middle_ub", activation="sigmoid")(fea_mid_ub)#
fea_mid_lb = Conv2D(512, (3, 3), padding='same', activation='relu')(fea_mid)
#fea_mid_lb = Flatten()(fea_mid_lb)
#fea_mid_lb = Dense(512, activation='relu')(fea_mid_lb)
fea_mid_lb = GlobalAveragePooling2D()(fea_mid_lb)
predictions_mid_lb = Dense(classes[3], name="middle_lb", activation="sigmoid")(fea_mid_lb)#
fea_mid_sh = Conv2D(512, (3, 3), padding='same', activation='relu')(fea_mid)
#fea_mid_sh = Flatten()(fea_mid_sh)
#fea_mid_sh = Dense(512, activation='relu')(fea_mid_sh)
fea_mid_sh = GlobalAveragePooling2D()(fea_mid_sh)
predictions_mid_sh = Dense(classes[4], name="middle_sh", activation="sigmoid")(fea_mid_sh)#
fea_mid_at = Conv2D(512, (3, 3), padding='same', activation='relu')(fea_mid)
#fea_mid_at = Flatten()(fea_mid_at)
#fea_mid_at = Dense(512, activation='relu')(fea_mid_at)
fea_mid_at = GlobalAveragePooling2D()(fea_mid_at)
predictions_mid_at = Dense(classes[5], name="middle_at", activation="sigmoid")(fea_mid_at)#
fea_mid_ot = Conv2D(512, (3, 3), padding='same', activation='relu')(fea_mid)
#fea_mid_ot = Flatten()(fea_mid_ot)
#fea_mid_ot = Dense(512, activation='relu')(fea_mid_ot)
fea_mid_ot = GlobalAveragePooling2D()(fea_mid_ot)
predictions_mid_ot = Dense(classes[6], name="middle_ot", activation="sigmoid")(fea_mid_ot)#
fea_hig = Conv2D(1024, (3, 3), padding='same', activation='relu')(fea_hig)
#fea_hig = Flatten()(fea_hig)
#fea_hig = Dense(512, activation='relu')(fea_hig)
fea_hig = GlobalAveragePooling2D()(fea_hig)
predictions_hig = Dense(classes[7], name="high_fea", activation="sigmoid")(fea_hig)
#"""
"""PCM2018"""
#predictions_hig = Dense(classes[2], activation="sigmoid", name="high")(concatenate([fea_low, fea_mid, fea_hig], axis=1))
"""PCM2018"""
predictions_priori = concatenate([predictions_low, predictions_mid_hs, predictions_mid_ub, predictions_mid_lb, predictions_mid_sh, predictions_mid_at, predictions_mid_ot], axis=1)
"""mar"""
#val = np.load("../results/state_transition_matrix.npy")
#state_transition_matrix = K.variable(value=val, dtype='float32', name='state_transition_matrix')
#predictions_hig_cond = Lambda(lambda x:K.dot(x, state_transition_matrix), name="high_cond")(predictions_priori)
"""mar"""
predictions_hig_cond = Dense(classes[7], name="high_cond", activation="sigmoid")(predictions_priori)#
#predictions_priori = K.reshape(concatenate([predictions_low, predictions_mid], axis=1), (-1, classes[0]+classes[1], 1))
#predictions_hig_cond = LSTM(classes[2], activation="sigmoid", name="high_cond")(predictions_priori)
predictions_hig_posterior = Lambda(lambda x:x[1] * x[0], name="high")([predictions_hig_cond, predictions_hig])
#predictions_hig_posterior = Lambda(lambda x:K.sigmoid(K.tanh((x[1] - 0.5) * np.pi) * x[0]), name="high")([predictions_hig_cond, predictions_hig])
#multi#Lambda(lambda x:x[0] * x[1], name="high_post")([predictions_hig_cond, predictions_hig])
#cond#Dense(classes[2], activation="sigmoid", name="high_post")(concatenate([predictions_hig, predictions_hig_cond], axis=1))
#add#Lambda(lambda x:(x[0] + x[1])/2, name="high_post")([predictions_hig_cond, predictions_hig])
""""mar"""
#predictions_low = Activation("sigmoid")(predictions_low)
#predictions_mid = Activation("sigmoid")(predictions_mid)
#predictions_hig_posterior = Activation("sigmoid")(predictions_hig_posterior)
"""mar"""
#predictions = concatenate([predictions_low, predictions_mid, predictions_hig_posterior], axis=1)
"""PCM2018"""
#predictions = concatenate([predictions_low, predictions_mid, predictions_hig], axis=1)
"""PCM2018"""
"""
predictions_low = Dense(classes[0], activation="sigmoid", name="low")(fea_low)
predictions_mid_fea = Dense(classes[1], activation="sigmoid", name="middle_fea")(fea_mid)
predictions_mid_cond = Dense(classes[1], activation="sigmoid", name="middle_cond")(predictions_low)
predictions_mid = Lambda(lambda x:(x[0] + x[1])/2, name="mid")([predictions_mid_fea, predictions_mid_cond])
predictions_hig_fea = Dense(classes[2], activation="sigmoid", name="high_fea")(fea_hig)
predictions_priori = concatenate([predictions_low, predictions_mid], axis=1)
predictions_hig_cond = Dense(classes[2], activation="sigmoid", name="high_cond")(predictions_priori)
predictions_hig = Lambda(lambda x:(x[0] + x[1])/2, name="high_post")([predictions_hig_cond, predictions_hig_fea])
predictions = concatenate([predictions_low, predictions_mid, predictions_hig], axis=1)
"""
"""
x = concatenate([spp_low, spp_mid, spp_hig], axis=1)#2048
#x = AveragePooling2D(pool_size=(7,7), strides=(7,7), padding='same')(x)
x = Dropout(0.4)(x)
x = Dense(2048, activation='relu')(x)
x = Dense(classes, activation='softmax')(x)
"""
# create the model
model = Model(inpt, [predictions_low, predictions_mid_hs, predictions_mid_ub, predictions_mid_lb, predictions_mid_sh, predictions_mid_at, predictions_mid_ot, predictions_hig_posterior], name='inception')
if weights == "imagenet":
weights = np.load("../results/googlenet_weights.npy", encoding='latin1').item()
for layer in model.layers:
if layer.get_weights() == []:
continue
#weight = layer.get_weights()
if layer.name in weights:
#print(layer.name, end=':')
#print(layer.get_weights()[0].shape == weights[layer.name]['weights'].shape, end=' ')
#print(layer.get_weights()[1].shape == weights[layer.name]['biases'].shape)
layer.set_weights([weights[layer.name]['weights'], weights[layer.name]['biases']])
# return the constructed network architecture
return model
def build_model(self):
timer = Timer()
timer.start()
print('[Model] Creating model..')
# this model is not sequencial
self.model = None
configs = self.c
for layer in configs['model']['layers']:
neurons = layer['neurons'] if 'neurons' in layer else None
dropout_rate = layer['rate'] if 'rate' in layer else None
activation = layer['activation'] if 'activation' in layer else None
input_timesteps = layer[
'input_timesteps'] if 'input_timesteps' in layer else None
input_features = layer[
'input_features'] if 'input_features' in layer else None
filters = layer['filters'] if 'filters' in layer else None
kernel_size = layer[
'kernel_size'] if 'kernel_size' in layer else None
pool_size = layer['pool_size'] if 'pool_size' in layer else None
#print('input_features %s input_timesteps %s ' % ( input_features, input_timesteps))
if layer['type'] == 'cnn':
first_input = Input(shape=(input_timesteps, input_features))
first_output = Conv1D(
filters=filters,
kernel_size=kernel_size,
activation=activation,
input_shape=(input_timesteps, input_features))(first_input)
second_input = Input(shape=(input_timesteps, input_features))
second_output = Conv1D(
filters=filters,
kernel_size=kernel_size,
activation=activation,
input_shape=(input_timesteps,
input_features))(second_input)
third_input = Input(shape=(input_timesteps, input_features))
third_output = Conv1D(
filters=filters,
kernel_size=kernel_size,
activation=activation,
input_shape=(input_timesteps, input_features))(third_input)
output = concatenate(
[first_output, second_output, third_output])
if layer['type'] == 'dense':
output = Dense(neurons, activation=activation)(output)
if layer['type'] == 'dropout':
output = Dropout(dropout_rate)(output)
if layer['type'] == 'maxpooling':
output = MaxPooling1D(pool_size=pool_size)(output)
if layer['type'] == 'flatten':
output = Flatten()(output)
if layer['type'] == 'activation':
output = Activation('linear')(output)
self.model = Model(inputs=[first_input, second_input, third_input],
outputs=output)
self.model.compile(loss=configs['model']['loss'],
optimizer=configs['model']['optimizer'],
metrics=configs['model']['metrics'])
print(self.model.summary())
print('[Model] Model Compiled with structure:', self.model.inputs)
timer.stop()
def getResNet(self,
outputClasses,
modelInput,
loss,
optimizer,
metrics,
batchNormalization=False):
initialX = layers.Conv2D(64, kernel_size=(6, 6),
padding='same')(modelInput)
initialX = layers.ReLU()(initialX)
if batchNormalization:
initialX = layers.BatchNormalization()(initialX)
x = layers.MaxPooling2D(pool_size=(2, 2), strides=(1, 1),
name='xPool')(initialX)
x2 = layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name='x2Pool')(initialX)
x3 = layers.MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
name='x3Pool')(initialX)
# stage 1x
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same',
name='xEntry')(x)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x = layers.Add(name='stage1x')([x, stage])
# stage 2x
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(x)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x = layers.Add(name='stage2x')([x, stage])
# stage 3x
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(x)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x = layers.Add(name='stage3x')([x, stage])
# stage 1x2
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same',
name='x2Entry')(x2)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x2 = layers.Add(name='stage1x2')([x2, stage])
# stage 2x2
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(x2)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x2 = layers.Add(name='stage2x2')([x2, stage])
# stage 3x2
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(x2)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x2 = layers.Add(name='stage3x2')([x2, stage])
# stage 1x3
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same',
name='x3Entry')(x3)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x3 = layers.Add(name='stage1x3')([x3, stage])
# stage 2x3
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(x3)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x3 = layers.Add(name='stage2x3')([x3, stage])
# stage 3x3
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(x3)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(32,
kernel_size=(3, 3),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
stage = layers.Conv2D(64,
kernel_size=(1, 1),
activation=activations.relu,
padding='same')(stage)
if batchNormalization:
stage = layers.BatchNormalization()(stage)
x3 = layers.Add(name='stage3x3')([x3, stage])
x = layers.MaxPool2D(pool_size=(2, 2))(x)
x2 = layers.MaxPool2D(pool_size=(2, 2))(x2)
x3 = layers.MaxPool2D(pool_size=(2, 2))(x3)
x = layers.Flatten(name='outputx')(x)
x2 = layers.Flatten(name='outputx2')(x2)
x3 = layers.Flatten(name='outputx3')(x3)
mergedX = layers.concatenate(inputs=[x, x2, x3], name='mergedX')
mergedX = layers.Dense(500)(mergedX)
mergedX = layers.ReLU()(mergedX)
mergedX = layers.Dropout(0.1)(mergedX)
mergedX = layers.Dense(outputClasses,
activation=activations.softmax,
name='output')(mergedX)
if batchNormalization:
model = models.Model(modelInput, mergedX, name="ResNetBN")
else:
model = models.Model(modelInput, mergedX, name="ResNet")
# model.summary()
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
return model
# cnn_filter_sizes 3
# cnn_filters 32
# cnn_pool 0
# cnn_dilation 2
# cnn_dense 1
cd1 = Conv1D(32,
kernel_size=3,
strides=strides,
activation=None,
padding='same',
dilation_rate=2)(p5)
bd1 = BatchNormalization()(cd1)
ad1 = Activation('relu')(bd1)
dd1 = Dropout(conv_dropout)(ad1)
od1 = concatenate([p5, dd1])
# cnn_filter_sizes 3
# cnn_filters 32
# cnn_pool 0
# cnn_dilation 4
# cnn_dense 1
cd2 = Conv1D(32,
kernel_size=3,
strides=strides,
activation=None,
padding='same',
dilation_rate=4)(od1)
bd2 = BatchNormalization()(cd2)
ad2 = Activation('relu')(bd2)
dd2 = Dropout(conv_dropout)(ad2)
layer.trainable = False
for layer in vgg_conv_2_base.layers:
layer.name += "_1"
x1 = vgg_conv_1_base(g1)
x2 = vgg_conv_1_base(g2)
x1 = MaxPooling2D((2,2))(x1)
x2 = MaxPooling2D((2,2))(x2)
rs1 = Reshape((1,4608))(x1)
rs2 = Reshape((1,4608))(x2)
c = concatenate([rs1,rs2],axis=1)
c = Dropout(0.5)(c)
r1 = LSTM(512,return_sequences=True,activation='linear', recurrent_dropout=0.5, kernel_regularizer=l2(l2_val))(c)
r1 = PReLU()(r1)
r1 = Dropout(0.5)(r1)
r2 = LSTM(512,activation='linear',recurrent_dropout=0.5, dropout=0.5, kernel_regularizer=l2(l2_val))(r1)
r2 = PReLU()(r2)
output = Dense(1)(r2)
def wnet_model(dataset, n_classes=5, im_sz=160, n_channels=3, n_filters_start=32, growth_factor=2):
droprate=0.25
#-------------Encoder
#Block1
n_filters = n_filters_start
inputs = Input((im_sz, im_sz, n_channels), name='input')
#inputs = BatchNormalization()(inputs)
conv1 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv1_1')(inputs)
actv1 = LeakyReLU(name = 'actv1_1')(conv1)
conv1 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv1_2')(actv1)
actv1 = LeakyReLU(name = 'actv1_2')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2), name = 'maxpool1')(actv1)
#pool1 = Dropout(droprate)(pool1)
#Block2
n_filters *= growth_factor
pool1 = IC(pool1)
conv2 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv2_1')(pool1)
actv2 = LeakyReLU(name = 'actv2_1')(conv2)
conv2 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv2_2')(actv2)
actv2 = LeakyReLU(name = 'actv2_2')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2), name = 'maxpool2')(actv2)
#Block3
n_filters *= growth_factor
pool2 = IC(pool2)
conv3 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv3_1')(pool2)
actv3 = LeakyReLU(name = 'actv3_1')(conv3)
conv3 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv3_2')(actv3)
actv3 = LeakyReLU(name = 'actv3_2')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2), name = 'maxpool3')(actv3)
#Block4
n_filters *= growth_factor
pool3 = IC(pool3)
conv4_0 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv4_1')(pool3)
actv4_0 = LeakyReLU(name = 'actv4_1')(conv4_0)
conv4_0 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv4_0_2')(actv4_0)
actv4_0 = LeakyReLU(name = 'actv4_2')(conv4_0)
pool4_1 = MaxPooling2D(pool_size=(2, 2), name = 'maxpool4')(actv4_0)
#Block5
n_filters *= growth_factor
pool4_1 = IC(pool4_1)
conv4_1 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv5_1')(pool4_1)
actv4_1 = LeakyReLU(name = 'actv5_1')(conv4_1)
conv4_1 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv5_2')(actv4_1)
actv4_1 = LeakyReLU(name = 'actv5_2')(conv4_1)
pool4_2 = MaxPooling2D(pool_size=(2, 2), name = 'maxpool5')(actv4_1)
#Block6
n_filters *= growth_factor
conv5 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv6_1')(pool4_2)
actv5 = LeakyReLU(name = 'actv6_1')(conv5)
conv5 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv6_2')(actv5)
actv5 = LeakyReLU(name = 'actv6_2')(conv5)
#-------------Decoder
#Block7
n_filters //= growth_factor
up6_1 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same', name = 'up7')(actv5), actv4_1], name = 'concat7')
up6_1 = IC(up6_1)
conv6_1 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv7_1')(up6_1)
actv6_1 = LeakyReLU(name = 'actv7_1')(conv6_1)
conv6_1 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv7_2')(actv6_1)
conv6_1 = LeakyReLU(name = 'actv7_2')(conv6_1)
#Block8
n_filters //= growth_factor
up6_2 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same', name = 'up8')(conv6_1), actv4_0], name = 'concat8')
up6_2 = IC(up6_2)
conv6_2 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv8_1')(up6_2)
actv6_2 = LeakyReLU(name = 'actv8_1')(conv6_2)
conv6_2 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv8_2')(actv6_2)
conv6_2 = LeakyReLU(name = 'actv8_2')(conv6_2)
#Block9
n_filters //= growth_factor
up7 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same', name = 'up9')(conv6_2), actv3], name = 'concat9')
up7 = IC(up7)
conv7 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv9_1')(up7)
actv7 = LeakyReLU(name = 'actv9_1')(conv7)
conv7 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv9_2')(actv7)
conv7 = LeakyReLU(name = 'actv9_2')(conv7)
#Block10
n_filters //= growth_factor
up8 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same', name = 'up10')(conv7), actv2], name = 'concat10')
up8 = IC(up8)
conv8 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv10_1')(up8)
actv8 = LeakyReLU(name = 'actv10_1')(conv8)
conv8 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv10_2')(actv8)
conv8 = LeakyReLU(name = 'actv10_2')(conv8)
#Block11
n_filters //= growth_factor
up9 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same', name = 'up11')(conv8), actv1], name = 'concat11')
conv9 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv11_1')(up9)
actv9 = LeakyReLU(name = 'actv11_1')(conv9)
conv9 = Conv2D(n_filters, (3, 3), padding='same', name = 'conv11_2')(actv9)
actv9 = LeakyReLU(name = 'actv11_2')(conv9)
output1 = Conv2D(n_classes, (1, 1), activation='softmax', name = 'output1')(actv9)
#-------------Second UNet
#-------------Encoder
#Block12
conv10 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(output1)
actv10 = LeakyReLU()(conv10)
conv10 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv10)
actv10 = LeakyReLU()(conv10)
pool10 = MaxPooling2D(pool_size=(2, 2))(actv10)
#Block13
n_filters *= growth_factor
pool10 = IC(pool10)
#Bridge
pool10 = concatenate([pool10, conv8])
conv11 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(pool10)
actv11 = LeakyReLU()(conv11)
conv11 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv11)
actv11 = LeakyReLU()(conv11)
pool11 = MaxPooling2D(pool_size=(2, 2))(actv11)
#Block14
n_filters *= growth_factor
pool11 = IC(pool11)
pool11 = concatenate([pool11, conv7])
conv12 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(pool11)
actv12 = LeakyReLU()(conv12)
conv12 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv12)
actv12 = LeakyReLU()(conv12)
pool12 = MaxPooling2D(pool_size=(2, 2))(actv12)
#Block15
n_filters *= growth_factor
pool12 = IC(pool12)
pool12 = concatenate([pool12, conv6_2])
conv13_0 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(pool12)
actv13_0 = LeakyReLU()(conv13_0)
conv13_0 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv13_0)
actv13_0 = LeakyReLU()(conv13_0)
pool13_1 = MaxPooling2D(pool_size=(2, 2))(actv13_0)
#Block16
n_filters *= growth_factor
pool13_1 = IC(pool13_1)
pool13_1 = concatenate([pool13_1, conv6_1])
conv13_1 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(pool13_1)
actv13_1 = LeakyReLU()(conv13_1)
conv13_1 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv13_1)
actv13_1 = LeakyReLU()(conv13_1)
pool13_2 = MaxPooling2D(pool_size=(2, 2))(actv13_1)
#Block17
n_filters *= growth_factor
pool13_2 = concatenate([pool13_2, actv5])
conv14 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(pool13_2)
actv14 = LeakyReLU()(conv14)
conv14 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv14)
actv14 = LeakyReLU()(conv14)
#-------------Decoder
#Block18
n_filters //= growth_factor
#Skip
up15_1 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same')(actv14), Add()([actv13_1, actv4_1])])
up15_1 = IC(up15_1)
conv15_1 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(up15_1)
actv15_1 = LeakyReLU()(conv15_1)
conv15_1 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv15_1)
conv15_1 = LeakyReLU()(conv15_1)
#Block19
n_filters //= growth_factor
#Skip
up15_2 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same')(conv15_1), Add()([actv13_0, actv4_0])])
up15_2 = IC(up15_2)
conv15_2 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(up15_2)
actv15_2 = LeakyReLU()(conv15_2)
conv15_2 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv15_2)
conv15_2 = LeakyReLU()(conv15_2)
#Block20
n_filters //= growth_factor
#Skip
up16 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same')(conv15_2), Add()([actv12, actv3])])
up16 = IC(up16)
conv16 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(up16)
actv16 = LeakyReLU()(conv16)
conv16 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv16)
conv16 = LeakyReLU()(conv16)
#Block21
n_filters //= growth_factor
#Skip
up17 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same')(conv16), Add()([actv11, actv2])])
up17 = IC(up17)
conv17 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(up17)
actv17 = LeakyReLU()(conv17)
conv17 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv17)
conv17 = LeakyReLU()(conv17)
#Block22
n_filters //= growth_factor
#Skip
up18 = concatenate([Conv2DTranspose(n_filters, (2, 2), strides=(2, 2), padding='same')(conv17), Add()([actv10, actv1])])
conv18 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(up18)
actv18 = LeakyReLU()(conv18)
conv18 = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer = 'he_uniform', bias_initializer = 'he_uniform')(actv18)
actv18 = LeakyReLU()(conv18)
#conv19 = Conv2D(n_classes, (1, 1), activation='sigmoid')(actv18)
conv19 = Conv2D(n_channels, (1, 1), activation='sigmoid', name = 'output2')(actv18)
output2 = conv19
model = Model(inputs=inputs, outputs=[output1, output2])
def mean_squared_error(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
return K.mean(K.square(y_pred_f - y_true_f))
#n_instances_per_class = [v for k, v in get_n_instances(dataset).items()]
model.compile(optimizer=Adam(lr = 10e-5), loss=[dice_coef_multilabel, mean_squared_error], loss_weights = [0.95, 0.05])
#model.compile(optimizer=Adam(lr = 10e-5), loss=[dice_coef_multilabel, mix], loss_weights = [0.95, 0.05])
#model.compile(optimizer=Adam(lr = 10e-5), loss=[categorical_class_balanced_focal_loss(n_instances_per_class, 0.99), mean_squared_error], loss_weights = [0.95, 0.05])
return model
name='lstmdense',
kernel_initializer='lecun_uniform')(drop)
norm = BatchNormalization(momentum=0.6, name='lstmdense_norm')(dense)
for i in xrange(1, 5):
dense = Dense(50, activation='relu', name='dense%i' % i)(norm)
norm = BatchNormalization(momentum=0.6, name='dense%i_norm' % i)(dense)
if LEARNMASS or LEARNPT or LEARNRHO:
to_merge = [norm]
if LEARNMASS:
to_merge.append(mass_inputs)
if LEARNRHO:
to_merge.append(rho_inputs)
if LEARNPT:
to_merge.append(pt_inputs)
merge = concatenate(to_merge)
dense = Dense(50, activation='tanh', name='dense5a')(merge)
norm = BatchNormalization(momentum=0.6, name='dense5a_norm')(dense)
# dense = Dense(50, activation='tanh', name='dense5')(norm)
# norm = BatchNormalization(momentum=0.6,name='dense5_norm')(dense)
else:
dense = Dense(50, activation='tanh', name='dense5')(norm)
norm = BatchNormalization(momentum=0.6, name='dense5_norm')(dense)
y_hat = Dense(config.n_truth, activation='softmax')(norm)
i = [inputs]
if LEARNMASS:
i.append(mass_inputs)
if LEARNRHO:
i.append(rho_inputs)
def build_model(self, n_features):
"""
The method builds a new member of the ensemble and returns it.
"""
# derived parameters
self.hyperparameters['n_members'] = self.hyperparameters[
'n_segments'] * self.hyperparameters['n_members_segment']
# initialize optimizer and early stopping
self.optimizer = Adam(lr=self.hyperparameters['lr'],
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.,
amsgrad=False)
self.es = EarlyStopping(monitor=f'val_{self.loss_name}',
min_delta=0.0,
patience=self.hyperparameters['patience'],
verbose=1,
mode='min',
restore_best_weights=True)
inputs = Input(shape=(n_features, ))
h = GaussianNoise(self.hyperparameters['noise_in'],
name='noise_input')(inputs)
for i in range(self.hyperparameters['layers']):
h = Dense(self.hyperparameters['neurons'],
activation=self.hyperparameters['activation'],
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_hidden'],
self.hyperparameters['l2_hidden']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name=f'hidden_{i}')(h)
h = Dropout(self.hyperparameters['dropout'],
name=f'hidden_dropout_{i}')(h)
mu = Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_mu'],
self.hyperparameters['l2_mu']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='mu_output')(h)
mu = GaussianNoise(self.hyperparameters['noise_mu'],
name='noise_mu')(mu)
if self.hyperparameters['pdf'] == 'normal' or self.hyperparameters[
'pdf'] == 'skewed':
sigma = Dense(1,
activation='softplus',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_sigma'],
self.hyperparameters['l2_sigma']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='sigma_output')(h)
sigma = GaussianNoise(self.hyperparameters['noise_sigma'],
name='noise_sigma')(sigma)
if self.hyperparameters['pdf'] == 'skewed':
alpha = Dense(1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
self.hyperparameters['l1_alpha'],
self.hyperparameters['l2_alpha']),
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='alpha_output')(h)
alpha = GaussianNoise(self.hyperparameters['noise_alpha'],
name='noise_alpha')(alpha)
if self.hyperparameters['pdf'] is None:
outputs = mu
elif self.hyperparameters['pdf'] == 'normal':
outputs = concatenate([mu, sigma])
elif self.hyperparameters['pdf'] == 'skewed':
outputs = concatenate([mu, sigma, alpha])
model = Model(inputs=inputs, outputs=outputs)
return model
def fetch_model_def(self):
input_protein_seq = Input(shape=(self.maxlen, ))
input_bio_feats = Input(shape=(self.num_bio_feats, ))
embedding_layer = Embedding(
self.vocab_size,
self.embedding_dim,
input_length=self.maxlen,
)
# Embed input to a continuous space
model = None
if self.protein_seq_feats is not None:
embedded = embedding_layer(input_protein_seq)
embedded = SpatialDropout1D(self.em_drop)(embedded)
# Attention after embedding
if self.attention is not None:
embedded = utils.attention_3d_block(embedding, embedded)
# Convolution Layers
x = embedded
if self.cnn_config is not None:
for k in range(len(self.cnn_config)):
convs = []
# Lists of feature maps and kernel sizes
# Check to see if we have more than one kernel
if ':' in self.feature_maps[k]:
feature_maps = [
int(f) for f in self.feature_maps[k].split(':')
]
kernel_sizes = [
int(f) for f in self.kernel_sizes[k].split(':')
]
else:
feature_maps = [int(self.feature_maps[k])]
kernel_sizes = [int(self.kernel_sizes[k])]
# An integer or a string
pool_size = self.max_pool_size[k]
for i in range(len(feature_maps)):
if k == len(self.cnn_config
) - 1 and self.rnn_config is None:
conv_layer = Conv1D(filters=feature_maps[i],
kernel_size=kernel_sizes[i],
padding='valid',
activation='relu',
strides=1)
else:
conv_layer = Conv1D(filters=feature_maps[i],
kernel_size=kernel_sizes[i],
padding='same',
activation='relu',
strides=1)
conv_out = conv_layer(x)
# Global can't be given as pool_size if RNN nis to be used after CNN. Need to be careful
if not pool_size.lower() == 'no':
if pool_size.lower(
) == 'global' and self.rnn_config is None:
conv_out = MaxPooling1D(
pool_size=conv_layer.output_shape[1])(
conv_out)
conv_out = Flatten()(conv_out)
else:
conv_out = MaxPooling1D(
pool_size=int(pool_size))(conv_out)
convs.append(conv_out)
# the below concat approach will be deprecated in 4 months
#x = merge(convs, mode='concat') if len(convs) > 1 else convs[0]
if len(convs) > 1:
x = convs[0]
for i in range(1, len(convs)):
x = concatenate([x, convs[i]])
else:
x = convs[0]
# Recurrent Layers
if self.rnn_config is not None:
for j in range(len(self.rnn_config)):
rnn_unit_type = self.rnn_config[j].split(',')[0]
rnn_units = int(self.rnn_config[j].split(',')[1])
rnn_drop = float(self.rnn_config[j].split(',')[2])
rnn_rec_drop = float(self.rnn_config[j].split(',')[3])
#rnn_out_drop = float(self.rnn_config[j].split(',')[4])
if rnn_unit_type.lower() == 'gru':
if j == len(self.rnn_config) - 1:
if self.rnn_bidirectional is None:
x = GRU(rnn_units,
dropout=rnn_drop,
recurrent_dropout=rnn_rec_drop)(x)
else:
x = Bidirectional(
GRU(rnn_units,
dropout=rnn_drop,
recurrent_dropout=rnn_rec_drop))(x)
else:
if self.rnn_bidirectional is None:
x = GRU(rnn_units,
dropout=rnn_drop,
recurrent_dropout=rnn_rec_drop,
return_sequences=True)(x)
else:
x = Bidirectional(
GRU(rnn_units,
dropout=rnn_drop,
recurrent_dropout=rnn_rec_drop,
return_sequences=True))(x)
elif rnn_unit_type.lower() == 'lstm':
if j == len(self.rnn_config) - 1:
if self.rnn_bidirectional is None:
x = LSTM(rnn_units,
dropout=rnn_drop,
recurrent_dropout=rnn_rec_drop)(x)
else:
x = Bidirectional(
LSTM(rnn_units,
dropout=rnn_drop,
recurrent_dropout=rnn_rec_drop))(x)
else:
if self.rnn_bidirectional is None:
x = LSTM(rnn_units,
dropout=rnn_drop,
recurrent_dropout=rnn_rec_drop,
return_sequences=True)(x)
else:
x = Bidirectional(
LSTM(rnn_units,
dropout=rnn_drop,
recurrent_dropout=rnn_rec_drop,
return_sequences=True))(x)
# Append biological feats
if self.biofeats is not None:
#x = merge([x, input_bio_feats], mode='concat')
x = concatenate([x, input_bio_feats])
# Dense Layers
for l in range(len(self.fc_config)):
fc_dim = int(self.fc_dims[l])
fc_dropout = float(self.fc_drop[l])
x = Dense(fc_dim)(x)
x = Dropout(fc_dropout)(x)
x = Activation('relu')(x)
main_output = Dense(self.num_classes, activation='sigmoid')(x)
if self.biofeats is not None:
model = Model(input=[input_protein_seq, input_bio_feats],
output=main_output)
else:
model = Model(input=input_protein_seq, output=main_output)
elif self.biofeats is not None:
# this will run if with only biological feats
x = input_bio_feats
for l in range(len(self.fc_config)):
fc_dim = int(self.fc_dims[l])
fc_dropout = float(self.fc_drop[l])
x = Dense(fc_dim)(x)
x = Dropout(fc_dropout)(x)
x = Activation('relu')(x)
main_output = Dense(self.num_classes, activation='sigmoid')(x)
model = Model(input=input_bio_feats, output=main_output)
return model
def Unet2(img_rows, img_cols, loss , optimizer, metrics, fc_size = 0, channels = 3):
filter_size = 5
filter_size_2 = 11
dropout_a = 0.5
dropout_b = 0.5
dropout_c = 0.5
gaussian_noise_std = 0.025
inputs = Input((img_rows, img_cols,channels))
input_with_noise = GaussianNoise(gaussian_noise_std)(inputs)
conv1 = Conv2D(32, filter_size, filter_size, activation='relu', padding='same')(input_with_noise)
conv1 = Conv2D(32, filter_size, filter_size, activation='relu', padding='same')(conv1)
conv1 = Conv2D(32, filter_size, filter_size, activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D((2, 2), strides=(2, 2))(conv1)
pool1 = GaussianNoise(gaussian_noise_std)(pool1)
conv2 = Conv2D(64, filter_size, filter_size, activation='relu', padding='same')(pool1)
conv2 = Conv2D(64, filter_size, filter_size, activation='relu', padding='same')(conv2)
conv2 = Conv2D(64, filter_size, filter_size, activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D((2, 2), strides=(2, 2))(conv2)
pool2 = GaussianNoise(gaussian_noise_std)(pool2)
conv3 = Conv2D(128, filter_size, filter_size, activation='relu', padding='same')(pool2)
conv3 = Conv2D(128, filter_size, filter_size, activation='relu', padding='same')(conv3)
conv3 = Conv2D(128, filter_size, filter_size, activation='relu', padding='same')(conv3)
pool3 = MaxPooling2D((2, 2), strides=(2, 2))(conv3)
pool3 = Dropout(dropout_a)(pool3)
if fc_size>0:
fc = Flatten()(pool3)
fc = Dense(fc_size)(fc)
fc = BatchNormalization()(fc)
fc = Activation('relu')(fc)
fc = Dropout(dropout_b)(fc)
n = img_rows/2/2/2
fc = Dense(img_rows*n*n)(fc)
fc = BatchNormalization()(fc)
fc = Activation('relu')(fc)
fc = GaussianNoise(gaussian_noise_std)(fc)
fc = Reshape((128,n,n))(fc)
else:
fc = Conv2D(256, filter_size, filter_size, activation='relu', padding='same')(pool3)
fc = BatchNormalization()(fc)
fc = Dropout(dropout_b)(fc)
up1 = concatenate([UpSampling2D(size=(2, 2))(fc), conv3], axis=1)
up1 = BatchNormalization()(up1)
up1 = Dropout(dropout_c)(up1)
conv4 = Conv2D(128, filter_size_2, filter_size_2, activation='relu', padding='same')(up1)
conv4 = Conv2D(128, filter_size, filter_size, activation='relu', padding='same')(conv4)
conv4 = Conv2D(64, filter_size, filter_size, activation='relu', padding='same')(conv4)
up2 = concatenate([UpSampling2D(size=(2, 2))(conv4), conv2], axis=1)
up2 = BatchNormalization()(up2)
up2 = Dropout(dropout_c)(up2)
conv5 = Conv2D(64, filter_size_2, filter_size_2, activation='relu', padding='same')(up2)
conv5 = Conv2D(64, filter_size, filter_size, activation='relu', padding='same')(conv5)
conv5 = Conv2D(32, filter_size, filter_size, activation='relu', padding='same')(conv5)
up3 = concatenate([UpSampling2D(size=(2, 2))(conv5), conv1], axis=1)
up3 = BatchNormalization()(up3)
up3 = Dropout(dropout_c)(up3)
conv6 = Conv2D(32, filter_size_2, filter_size_2, activation='relu', padding='same')(up3)
conv6 = Conv2D(32, filter_size, filter_size, activation='relu', padding='same')(conv6)
conv6 = Conv2D(32, filter_size, filter_size, activation='relu', padding='same')(conv6)
conv7 = Conv2D(1, 1, 1, activation='sigmoid')(conv6)
model = Model(input=inputs, output=conv7)
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
return model
def AlexNet(weights_path=None):
inputs = Input(shape=(227, 227, 3))
conv_1 = Convolution2D(96, (11, 11),
strides=(4, 4),
activation='relu',
name='conv_1')(inputs)
conv_2 = MaxPooling2D((3, 3), strides=(2, 2))(conv_1)
conv_2 = crosschannelnormalization(name="convpool_1")(conv_2)
conv_2 = ZeroPadding2D((2, 2))(conv_2)
conv_2 = concatenate([
Convolution2D(
128, (5, 5), activation="relu", name='conv_2_' + str(i + 1))(
splittensor(ratio_split=2, id_split=i)(conv_2))
for i in range(2)
],
axis=-1,
name="conv_2")
conv_3 = MaxPooling2D((3, 3), strides=(2, 2))(conv_2)
conv_3 = crosschannelnormalization()(conv_3)
conv_3 = ZeroPadding2D((1, 1))(conv_3)
conv_3 = Convolution2D(384, (3, 3), activation='relu',
name='conv_3')(conv_3)
conv_4 = ZeroPadding2D((1, 1))(conv_3)
conv_4 = concatenate([
Convolution2D(
192, (3, 3), activation="relu", name='conv_4_' + str(i + 1))(
splittensor(ratio_split=2, id_split=i)(conv_4))
for i in range(2)
],
axis=-1,
name="conv_4")
conv_5 = ZeroPadding2D((1, 1))(conv_4)
conv_5 = concatenate([
Convolution2D(
128, (3, 3), activation="relu", name='conv_5_' + str(i + 1))(
splittensor(ratio_split=2, id_split=i)(conv_5))
for i in range(2)
],
axis=-1,
name="conv_5")
dense_1 = MaxPooling2D((3, 3), strides=(2, 2), name="convpool_5")(conv_5)
dense_1 = Flatten(name="flatten")(dense_1)
dense_1 = Dense(4096, activation='relu', name='dense_1')(dense_1)
dense_2 = Dropout(0.5)(dense_1)
dense_2 = Dense(4096, activation='relu', name='dense_2')(dense_2)
dense_3 = Dropout(0.5)(dense_2)
dense_3 = Dense(1000, name='dense_3')(dense_3)
prediction = Activation("softmax", name="softmax")(dense_3)
model = Model(inputs=inputs, outputs=prediction)
if weights_path:
model.load_weights(weights_path)
if K.backend() == 'tensorflow':
convert_all_kernels_in_model(model)
return model
def VGG16(img_rows, img_cols, pretrained, freeze_pretrained, loss , optimizer, metrics, channels=3):
inputs = Input((img_rows, img_cols,channels))
pad1 = ZeroPadding2D((1, 1), input_shape=(img_rows, img_cols,channels))(inputs)
conv1 = Conv2D(64,(3,3), activation='relu', name='conv1_1')(pad1)
conv1 = ZeroPadding2D((1, 1))(conv1)
conv1 = Conv2D(64,(3,3), activation='relu', name='conv1_2')(conv1)
pool1 = MaxPooling2D((2, 2), strides=(2, 2))(conv1)
pad2 = ZeroPadding2D((1, 1))(pool1)
conv2 = Conv2D(128,(3,3), activation='relu', name='conv2_1')(pad2)
conv2 = ZeroPadding2D((1, 1))(conv2)
conv2 = Conv2D(128,(3,3), activation='relu', name='conv2_2')(conv2)
pool2 = MaxPooling2D((2, 2), strides=(2, 2))(conv2)
pad3 = ZeroPadding2D((1, 1))(pool2)
conv3 = Conv2D(256,(3,3), activation='relu', name='conv3_1')(pad3)
conv3 = ZeroPadding2D((1, 1))(conv3)
conv3 = Conv2D(256,(3,3), activation='relu', name='conv3_2')(conv3)
conv3 = ZeroPadding2D((1, 1))(conv3)
conv3 = Conv2D(256,(3,3), activation='relu', name='conv3_3')(conv3)
pool3 = MaxPooling2D((2, 2), strides=(2, 2))(conv3)
pad4 = ZeroPadding2D((1, 1))(pool3)
conv4 = Conv2D(512,(3,3), activation='relu', name='conv4_1')(pad4)
conv4 = ZeroPadding2D((1, 1))(conv4)
conv4 = Conv2D(512,(3,3), activation='relu', name='conv4_2')(conv4)
conv4 = ZeroPadding2D((1, 1))(conv4)
conv4 = Conv2D(512,(3,3), activation='relu', name='conv4_3')(conv4)
pool4 = MaxPooling2D((2, 2), strides=(2, 2))(conv4)
pad5 = ZeroPadding2D((1, 1))(pool4)
conv5 = Conv2D(512,(3,3), activation='relu', name='conv5_1')(pad5)
conv5 = ZeroPadding2D((1, 1))(conv5)
conv5 = Conv2D(512,(3,3), activation='relu', name='conv5_2')(conv5)
conv5 = ZeroPadding2D((1, 1))(conv5)
conv5 = Conv2D(512,(3,3), activation='relu', name='conv5_3')(conv5)
model = Model(input=inputs, output=conv5)
# load weights
if pretrained:
weights_path = VGG16_WEIGHTS_NOTOP
f = h5py.File(weights_path)
for k in range(f.attrs['nb_layers']):
if k >= (len(model.layers) - 1):
# ignore the last layers in the savefile
break
g = f['layer_{}'.format(k)]
weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
model.layers[k+1].set_weights(weights)
f.close()
print('ImageNet pre-trained weights loaded.')
if freeze_pretrained:
for layer in model.layers:
layer.trainable = False
dropout_val = 0.5
up6 = concatenate([UpSampling2D(size=(2, 2))(conv5), conv4], axis=1)
up6 = Dropout(dropout_val)(up6)
conv6 = Conv2D(256,(3,3), activation='relu', padding='same')(up6)
conv6 = Conv2D(256,(3,3), activation='relu', padding='same')(conv6)
up7 = concatenate([UpSampling2D(size=(2, 2))(conv6), conv3], axis=1)
up7 = Dropout(dropout_val)(up7)
conv7 = Conv2D(128,(3,3), activation='relu', padding='same')(up7)
conv7 = Conv2D(128,(3,3), activation='relu', padding='same')(conv7)
up8 = concatenate([UpSampling2D(size=(2, 2))(conv7), conv2], axis=1)
up8 = Dropout(dropout_val)(up8)
conv8 = Conv2D(64,(3,3), activation='relu', padding='same')(up8)
conv8 = Conv2D(64,(3,3), activation='relu', padding='same')(conv8)
up9 = concatenate([UpSampling2D(size=(2, 2))(conv8), conv1], axis=1)
up9 = Dropout(dropout_val)(up9)
conv9 = Conv2D(32,(3,3), activation='relu', padding='same')(up9)
conv9 = Conv2D(32,(3,3), activation='relu', padding='same')(conv9)
conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conv9)
model = Model(input=inputs, output=conv10)
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
return model
def inc3_mirror(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(512,512,3),
pooling=None,
classes=1):
channel_axis = 3
img_input = layers.Input(shape=input_shape)
inception3_end_layers = InceptionV3(weights=weights, input_tensor=img_input, input_shape=input_shape)
x = inception3_end_layers[0]
# decoder_mixed 9: 16 x 16 x 2048
for i in range(2):
branch1x1 = conv2d_transpose_bn(x, 320, 1, 1)
branch3x3 = conv2d_transpose_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_transpose_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_transpose_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate(
[branch3x3_1, branch3x3_2],
axis=channel_axis,
name='decoder_mixed9_' + str(i))
branch3x3dbl = conv2d_transpose_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_transpose_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_transpose_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_transpose_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate(
[branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis)
branch_pool = layers.AveragePooling2D(
(3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_transpose_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='decoder_mixed' + str(9 + i))
# decoder_mixed 8: 32 x 32 x 1280
branch3x3 = conv2d_transpose_bn(x, 192, 1, 1)
branch3x3 = conv2d_transpose_bn(branch3x3, 320, 3, 3,
strides=(2, 2), padding='same')
# strides=(2, 2), padding='valid')
branch7x7x3 = conv2d_transpose_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_transpose_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_transpose_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_transpose_bn(
branch7x7x3, 192, 3, 3, strides=(2, 2), padding='same')
# branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid')
branch_pool = layers.UpSampling2D()(x)
# branch_pool = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch7x7x3, branch_pool],
axis=channel_axis,
name='decoder_mixed8')
# Create skip layer
x = layers.concatenate([x, inception3_end_layers[1]], axis=-1)
# mixed 7: 32 x 32 x 768
branch1x1 = conv2d_transpose_bn(x, 192, 1, 1)
branch7x7 = conv2d_transpose_bn(x, 192, 1, 1)
branch7x7 = conv2d_transpose_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_transpose_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_transpose_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_transpose_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='decoder_mixed7')
# mixed 5, 6: 32 x 32 x 768
for i in range(2):
branch1x1 = conv2d_transpose_bn(x, 192, 1, 1)
branch7x7 = conv2d_transpose_bn(x, 160, 1, 1)
branch7x7 = conv2d_transpose_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_transpose_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_transpose_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D(
(3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_transpose_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='decoder_mixed' + str(5 + i))
# mixed 4: 32 x 32 x 768
branch1x1 = conv2d_transpose_bn(x, 192, 1, 1)
branch7x7 = conv2d_transpose_bn(x, 128, 1, 1)
branch7x7 = conv2d_transpose_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_transpose_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_transpose_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_transpose_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_transpose_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='decoder_mixed4')
# mixed 3: 64 x 64 x 768
branch3x3 = conv2d_transpose_bn(x, 384, 3, 3, strides=(2, 2), padding='same')
branch3x3dbl = conv2d_transpose_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_transpose_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_transpose_bn(
branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='same')
# branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid')
branch_pool = layers.UpSampling2D()(x)
x = layers.concatenate(
[branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='decoder_mixed3')
# Create skip layer
x = layers.concatenate([x, inception3_end_layers[2]], axis=-1)
# mixed 2: 128 x 128 x 256
branch1x1 = conv2d_transpose_bn(x, 64, 1, 1)
branch5x5 = conv2d_transpose_bn(x, 48, 1, 1)
branch5x5 = conv2d_transpose_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_transpose_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_transpose_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_transpose_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_transpose_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='decoder_mixed2')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_transpose_bn(x, 64, 1, 1)
branch5x5 = conv2d_transpose_bn(x, 48, 1, 1)
branch5x5 = conv2d_transpose_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_transpose_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_transpose_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_transpose_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_transpose_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='decoder_mixed1')
# mixed 0, 1, 2: 128 x 128 x 256
branch1x1 = conv2d_transpose_bn(x, 64, 1, 1)
branch5x5 = conv2d_transpose_bn(x, 48, 1, 1)
branch5x5 = conv2d_transpose_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_transpose_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_transpose_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_transpose_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_transpose_bn(branch_pool, 32, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='decoder_mixed0')
x = layers.UpSampling2D()(x)
# Create skip layer
x = layers.concatenate([x, inception3_end_layers[3]], axis=-1)
x = conv2d_transpose_bn(x, 192, 3, 3, padding='same')
x = conv2d_transpose_bn(x, 80, 1, 1, padding='same')
x = layers.UpSampling2D()(x)
# Create skip layer
x = layers.concatenate([x, inception3_end_layers[4]], axis=-1)
x = conv2d_transpose_bn(x, 64, 3, 3)
x = conv2d_transpose_bn(x, 32, 3, 3, padding='same')
x = conv2d_transpose_bn(x, 32, 3, 3, strides=(2, 2), padding='same')
# ngide
res = conv2d_transpose_bn(img_input, 32, 3, 3)
res = conv2d_transpose_bn(res, 32, 3, 3)
x = layers.concatenate([x, res], axis=-1)
x = conv2d_transpose_bn(x, 32, 3, 3)
x = conv2d_transpose_bn(x, 32, 3, 3, padding='same')
x = layers.Conv2D(classes,(1,1),activation="sigmoid", name="out_inc3_mirror")(x)
model = models.Model(img_input, x, name='inc3_mirror')
model.compile(optimizer=RMSprop(lr=0.0001), loss=dice_loss, metrics=[dice_coeff, false_negative, false_negative_pixel])
print(model.summary())
return model
sequence_input4 = Input(shape=(max_sequence_length1,), dtype='int32')
wo_ind = Embedding(len(word_index4) + 1,
embedding_dim,
input_length=max_sequence_length1,
trainable=True)(sequence_input4)
from keras.layers import LSTM
lstm_output_size = 64
wo_ind = LSTM(lstm_output_size, dropout=0.5,
return_sequences=True)(wo_ind) #97%
wo_ind=Dense(100,activation='relu')(wo_ind)
k = 12
output = concatenate([texture,wo_ind])
print("output:",output)
"""
LSTM的创建
"""
output = LSTM(lstm_output_size, dropout=0.5,
return_sequences=True)(output) #97%
category_caps = Length(name='out_caps')(output)
preds = Dense(1, activation='sigmoid')(category_caps)
print("training>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
model = Model(inputs=[ sequence_input2,sequence_input4], outputs=[preds])
model.summary()
def InceptionV3(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(512,512,3),
pooling=None,
classes=1):
"""Instantiates the Inception v3 architecture.
Optionally loads weights pre-trained on ImageNet.
Note that the data format convention used by the model is
the one specified in your Keras config at `~/.keras/keras.json`.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)` (with `channels_last` data format)
or `(3, 299, 299)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 139.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as `"imagenet"` with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
# input_shape = _obtain_input_shape(
# input_shape,
# default_size=299,
# min_size=139,
# data_format=backend.image_data_format(),
# require_flatten=False,
# weights=weights)
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if backend.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 3
x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='same')
x = conv2d_bn(x, 32, 3, 3, padding='same')
x = conv2d_bn(x, 64, 3, 3)
skip1 = x
x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = conv2d_bn(x, 80, 1, 1, padding='same')
x = conv2d_bn(x, 192, 3, 3, padding='same')
skip2 = x
x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
# mixed 0, 1, 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed0')
# mixed 1: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed1')
# mixed 2: 35 x 35 x 256
branch1x1 = conv2d_bn(x, 64, 1, 1)
branch5x5 = conv2d_bn(x, 48, 1, 1)
branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 64, 1, 1)
x = layers.concatenate(
[branch1x1, branch5x5, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed2')
skip3 = x
# mixed 3: 17 x 17 x 768
branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='same')
branch3x3dbl = conv2d_bn(x, 64, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
branch3x3dbl = conv2d_bn(
branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='same')
# branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid')
branch_pool = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.concatenate(
[branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed3')
# mixed 4: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 128, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 128, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed4')
# mixed 5, 6: 17 x 17 x 768
for i in range(2):
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 160, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 160, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D(
(3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(5 + i))
# mixed 7: 17 x 17 x 768
branch1x1 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(x, 192, 1, 1)
branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
branch7x7dbl = conv2d_bn(x, 192, 1, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
branch_pool = layers.AveragePooling2D((3, 3),
strides=(1, 1),
padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch7x7, branch7x7dbl, branch_pool],
axis=channel_axis,
name='mixed7')
skip4 = x
# mixed 8: 8 x 8 x 1280
branch3x3 = conv2d_bn(x, 192, 1, 1)
branch3x3 = conv2d_bn(branch3x3, 320, 3, 3,
strides=(2, 2), padding='same')
# strides=(2, 2), padding='valid')
branch7x7x3 = conv2d_bn(x, 192, 1, 1)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
branch7x7x3 = conv2d_bn(
branch7x7x3, 192, 3, 3, strides=(2, 2), padding='same')
# branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid')
branch_pool = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
# branch_pool = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = layers.concatenate(
[branch3x3, branch7x7x3, branch_pool],
axis=channel_axis,
name='mixed8')
# mixed 9: 8 x 8 x 2048
for i in range(2):
branch1x1 = conv2d_bn(x, 320, 1, 1)
branch3x3 = conv2d_bn(x, 384, 1, 1)
branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
branch3x3 = layers.concatenate(
[branch3x3_1, branch3x3_2],
axis=channel_axis,
name='mixed9_' + str(i))
branch3x3dbl = conv2d_bn(x, 448, 1, 1)
branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
branch3x3dbl = layers.concatenate(
[branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis)
branch_pool = layers.AveragePooling2D(
(3, 3), strides=(1, 1), padding='same')(x)
branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
x = layers.concatenate(
[branch1x1, branch3x3, branch3x3dbl, branch_pool],
axis=channel_axis,
name='mixed' + str(9 + i))
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = engine.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = models.Model(inputs, x, name='inception_v3')
# Load weights.
if weights == 'imagenet':
if include_top:
weights_path = keras_utils.get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
file_hash='9a0d58056eeedaa3f26cb7ebd46da564')
else:
weights_path = keras_utils.get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='bcbd6486424b2319ff4ef7d526e38f63')
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return [x, skip4, skip3, skip2, skip1]
def inc3_u(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(512,512,3),
pooling=None,
classes=1):
img_input = layers.Input(shape=input_shape)
inception3_end_layers = InceptionV3(input_tensor=img_input, input_shape=input_shape)
x = inception3_end_layers[0]
x = layers.UpSampling2D((2, 2))(x)
x = layers.concatenate([x, inception3_end_layers[1]], axis=3)
x = conv2d_transpose_bn(x, 512, 3, 3)
x_res = x
x = conv2d_transpose_bn(x, 512, 3, 3)
x = conv2d_transpose_bn(x, 512, 3, 3)
x = layers.add([x_res, x])
x = layers.Activation('relu')(x)
x = layers.UpSampling2D((2, 2))(x)
x = layers.concatenate([x, inception3_end_layers[2]], axis=3)
x = conv2d_transpose_bn(x, 256, 3, 3)
x_res = x
x = conv2d_transpose_bn(x, 256, 3, 3)
x = conv2d_transpose_bn(x, 256, 3, 3)
x = layers.add([x_res, x])
x = layers.Activation('relu')(x)
x = layers.UpSampling2D((2, 2))(x)
x = layers.concatenate([x, inception3_end_layers[3]], axis=3)
# 56
x = conv2d_transpose_bn(x, 128, 3, 3)
x_res = x
x = conv2d_transpose_bn(x, 128, 3, 3)
x = conv2d_transpose_bn(x, 128, 3, 3)
x = layers.add([x_res, x])
x = layers.Activation('relu')(x)
x = layers.UpSampling2D((2, 2))(x)
x = layers.concatenate([x, inception3_end_layers[4]], axis=3)
x = conv2d_transpose_bn(x, 64, 3, 3)
x_res = x
x = conv2d_transpose_bn(x, 64, 3, 3)
x = conv2d_transpose_bn(x, 64, 3, 3)
x = layers.add([x_res, x])
x = layers.Activation('relu')(x)
x = layers.UpSampling2D((2, 2))(x)
res = conv2d_bn(img_input, 32, 3, 3)
x_res = res
res = conv2d_bn(res, 32, 3, 3)
res = conv2d_bn(res, 32, 3, 3)
res = layers.add([x_res, res])
x = layers.Activation('relu')(x)
x = layers.concatenate([x, res], axis=3)
x = conv2d_transpose_bn(x, 32, 3, 3)
x_res = x
x = conv2d_transpose_bn(x, 32, 3, 3)
x = conv2d_transpose_bn(x, 32, 3, 3)
x = layers.add([x_res, x])
x = layers.Activation('relu')(x)
x = layers.Conv2D(classes,(1,1),activation="sigmoid", name="out_inc3_u_net")(x)
model = models.Model(img_input, x, name='inc3_u_net')
model.compile(optimizer=RMSprop(lr=0.0001), loss=dice_loss, metrics=[dice_coeff, false_negative, false_negative_pixel])
print(model.summary())
return model
def get_fcn8(flag):
img_rows = flag.image_height
img_cols = flag.image_width
lr = flag.initial_learning_rate
nClasses = 3
inputs = Input((img_rows, img_cols, 3))
x = Conv2D(32, (3, 3),
activation=None,
padding='same',
name='block1_conv1')(inputs)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3),
activation=None,
padding='same',
name='block1_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
p1 = x
# Block 2
x = Conv2D(64, (3, 3),
activation=None,
padding='same',
name='block2_conv1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3),
activation=None,
padding='same',
name='block2_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
p2 = x
# Block 3
x = Conv2D(128, (3, 3),
activation=None,
padding='same',
name='block3_conv1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3),
activation=None,
padding='same',
name='block3_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3),
activation=None,
padding='same',
name='block3_conv3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
p3 = x
# Block 4
x = Conv2D(256, (3, 3),
activation=None,
padding='same',
name='block4_conv1')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3),
activation=None,
padding='same',
name='block4_conv2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3),
activation=None,
padding='same',
name='block4_conv3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
p4 = x
# Block 5
# x = Conv2D(512, (3, 3), activation=None, padding='same', name='block5_conv1')(x)
# x = BatchNormalization()(x)
# x = Activation('relu')(x)
# x = Conv2D(512, (3, 3), activation=None, padding='same', name='block5_conv2')(x)
# x = BatchNormalization()(x)
# x = Activation('relu')(x)
# x = Conv2D(512, (3, 3), activation=None, padding='same', name='block5_conv3')(x)
# x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
# p5 = x
vgg = Model(inputs, x)
### pool4 (16,32,256) --> up1 (32,64,256)
o = p4
o = Conv2D(256, (5, 5), activation=None, padding='same')(o)
o = BatchNormalization()(o)
o = Activation('relu')(o)
o = Conv2D(256, (3, 3), activation=None, padding='same')(o)
o = BatchNormalization()(o)
o = Activation('relu')(o)
o = Conv2DTranspose(256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same')(o)
### pool3 (32,64,128) --> (32,64,256)
o2 = p3
o2 = Conv2D(256, (3, 3), activation=None, padding='same')(o2)
o2 = BatchNormalization()(o2)
o2 = Activation('relu')(o2)
### concat1 [(32,64,256), (32,64,256)]
o = concatenate([o, o2], axis=3)
o = Conv2D(256, (3, 3), padding='same')(o)
o = BatchNormalization()(o)
o = Activation('relu')(o)
### up2 (32,64,512) --> (64,128,128)
o = Conv2DTranspose(128,
kernel_size=(4, 4),
strides=(2, 2),
padding='same')(o)
o = Conv2D(128, (3, 3), padding='same')(o)
o = BatchNormalization()(o)
o = Activation('relu')(o)
### pool2 (64,128,64) --> (64,128,128)
o2 = p2
o2 = Conv2D(128, (3, 3), activation=None, padding='same')(o2)
o2 = BatchNormalization()(o2)
o2 = Activation('relu')(o2)
### concat2 [(64,128,128), (64,128,128)]
o = concatenate([o, o2], axis=3)
### up3 (64,128,256) --> (128,256,64)
o = Conv2DTranspose(64, kernel_size=(2, 2), strides=(2, 2),
padding='same')(o)
o = Conv2D(64, (3, 3), padding='same')(o)
o = BatchNormalization()(o)
o = Activation('relu')(o)
### pool1 (128,256,64) --> (128,256,32)
o2 = p1
o2 = Conv2D(32, (3, 3), activation=None, padding='same')(o2)
o2 = BatchNormalization()(o2)
o2 = Activation('relu')(o2)
### concat3 [(128,256,64), (128,256,32)] --> (128,256,32)
o = concatenate([o, o2], axis=3)
o = Conv2D(32, (3, 3), padding='same')(o)
o = BatchNormalization()(o)
o = Activation('relu')(o)
### up (128,256,32) --> (256,512,32)
o = Conv2DTranspose(32, kernel_size=(2, 2), strides=(2, 2),
padding='same')(o)
### mask out (128,256,32) --> (256,512,3)
o = Conv2D(nClasses, (3, 3), padding='same')(o)
o = Activation('sigmoid')(o)
model = Model(inputs=[inputs], outputs=[o])
model.compile(optimizer=Adam(lr=lr, decay=1e-6),
loss=pixelwise_binary_ce,
metrics=[dice_coef])
return model
def train_model(data_file, job_dir, context_model_file, data_bucket,
training_epochs, **args):
print('Tensorflow version: ' + tf.__version__)
print('Keras version: ' + keras.__version__)
print('Data bucket: ' + data_bucket)
print('Data file: ' + data_file)
print('Context model file: ' + context_model_file)
print('Job directory: ' + job_dir)
print('Training epochs: ' + str(training_epochs))
## Prepare training data
img_height = 224
img_width = 224
batch_size = 16
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
train_set_key = 'train_set_'
valid_set_key = 'valid_set_'
classes_key = 'list_classes'
print('Trying to read file: ' + data_bucket + data_file)
with file_io.FileIO(data_bucket + data_file, mode='rb') as input_f:
print('Opened file')
with file_io.FileIO(data_file, mode='wb+') as output_f:
output_f.write(input_f.read())
print('Written file to: ' + data_file)
print('Copied ' + data_file + ' locally')
# Read h5 dataset file
dataset = h5py.File(data_file, 'r')
list_classes = dataset[classes_key]
classes = list_classes.shape[0]
train_set_x = dataset[train_set_key + 'x']
valid_set_x = dataset[valid_set_key + 'x']
print('Converted train/validation set to categorical')
train_set_y = np_utils.to_categorical(dataset[train_set_key + 'y'],
classes)
valid_set_y = np_utils.to_categorical(dataset[valid_set_key + 'y'],
classes)
print('Read datasets from h5 file')
# Create data generator
datagen = ImageDataGenerator(rescale=1. / 255)
## Give same input to both models
model_input = Input(shape=input_shape)
print('Trying to read file: ' + data_bucket + context_model_file)
with file_io.FileIO(data_bucket + context_model_file,
mode='rb') as input_f:
print('Opened file')
with file_io.FileIO(context_model_file, mode='wb+') as output_f:
output_f.write(input_f.read())
print('Written file to: ' + context_model_file)
print('Copied ' + context_model_file + ' locally')
## Load context recognition model pre-trained on 27 categories of Places205
loaded_context_model = load_model(context_model_file)
# Construct new context CNN model with pre-trained weights loaded but connected to input tensor
context_cnn = DenseNet121(weights=None,
include_top=False,
input_tensor=model_input)
x = GlobalAveragePooling2D()(context_cnn.output)
x = Dense(1024, activation='relu')(x)
context_predictions = Dense(27, activation='softmax')(x)
context_model = Model(inputs=model_input, outputs=context_predictions)
for new_layer, layer in zip(context_model.layers[1:],
loaded_context_model.layers[1:]):
new_layer.set_weights(layer.get_weights())
## Load object recognition model pre-trained on ImageNet
object_model = ResNet50(include_top=True,
weights='imagenet',
input_tensor=model_input,
pooling=None,
classes=1000)
# Hack to avoid https://github.com/keras-team/keras/issues/3974
for i in range(1, len(object_model.layers)):
object_model.layers[i].name = object_model.layers[i].name + '1'
## Merge/concatenate outputs of the two sub-models
context_output = context_model.output
object_output = object_model.output
y = concatenate([context_output, object_output])
## Add extra layer(s) for classification of activities
y = Dense(1024, activation='relu')(y)
predictions = Dense(classes, activation='softmax')(y)
## Set C-CNN layers to be not trainable
for layer in context_model.layers:
layer.trainable = False
## Compile model
model = Model(inputs=model_input, outputs=predictions)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_filename = 'activity-recognition-model.h5'
checkpoint = ModelCheckpoint(model_filename,
verbose=1,
monitor='val_acc',
save_best_only=True,
mode='auto')
print('Starting fit_generator on model')
# Fits the model on batches
model.fit_generator(datagen.flow(train_set_x,
train_set_y,
batch_size=batch_size),
steps_per_epoch=len(train_set_x) / batch_size,
epochs=training_epochs,
validation_data=datagen.flow(valid_set_x,
valid_set_y,
batch_size=batch_size),
validation_steps=len(valid_set_x) / batch_size,
callbacks=[checkpoint])
print('Finished training only the extra layers')
# # Save the model locally
# model.save(model_filename)
# print ('Saved model to :' + model_filename)
# Save the model to the Cloud Storage bucket's jobs directory
with file_io.FileIO(model_filename, mode='rb') as input_f:
with file_io.FileIO(job_dir + '/' + model_filename,
mode='wb+') as output_f:
output_f.write(input_f.read())
def ssd_model(input_shape, num_classes=21):
"""SSD300 + Resnet50 architecture.
# Arguments
input_shape: Shape of the input image, expected to be (300, 300, 3).
num_classes: Number of classes including background.
# References
https://arxiv.org/abs/1512.02325
"""
net = {}
img_size = (input_shape[1], input_shape[0])
input_tensor = Input(shape=input_shape)
net['input'] = input_tensor
resnet_head = ResNet50(include_top=False, input_tensor=input_tensor)
activation_layer_names = []
for layer in resnet_head.layers:
if layer._outbound_nodes:
net[layer.name] = layer.output
layer.trainable = False
if layer.name.startswith('activation'):
activation_layer_names.append(layer.name)
# The ResNet50 architecture changed in June 2018 to exclude
# an average pooling layer when include_top is false. This caused
# the last activation layer to no longer have output nodes, so
# we add it to the list of activation layers here if it is missing.
if 'activation_49' not in activation_layer_names:
net['activation_49'] = resnet_head.layers[-1].output
resnet_head.layers[-1].trainable = False
activation_layer_names.append('activation_49')
prev_size_layer_name = activation_layer_names[-10]
net['pool5'] = MaxPooling2D((3, 3),
strides=(1, 1),
padding='same',
name='pool5')(net[activation_layer_names[-1]])
# FC6
net['fc6'] = Conv2D(1024, (3, 3),
dilation_rate=(6, 6),
activation='relu',
padding='same',
name='fc6')(net['pool5'])
# FC7
net['fc7'] = Conv2D(1024, (1, 1),
activation='relu',
padding='same',
name='fc7')(net['fc6'])
# Block 6
net['conv6_1'] = _wrap_with_bn_and_dropout(net,
Conv2D(256, (1, 1),
activation='linear',
padding='same',
name='conv6_1')(
net['fc7']),
name='6_1')
net['conv6_2'] = Conv2D(512, (3, 3),
strides=(2, 2),
activation='relu',
padding='same',
name='conv6_2')(net['conv6_1'])
# Block 7
net['conv7_1'] = _wrap_with_bn_and_dropout(net,
Conv2D(128, (1, 1),
activation='linear',
padding='same',
name='conv7_1')(
net['conv6_2']),
name='7_1')
net['conv7_2'] = ZeroPadding2D()(net['conv7_1'])
net['conv7_2'] = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
name='conv7_2')(net['conv7_2'])
# Block 8
net['conv8_1'] = _wrap_with_bn_and_dropout(net,
Conv2D(128, (1, 1),
activation='relu',
padding='same',
name='conv8_1')(
net['conv7_2']),
name='8_1')
net['conv8_2'] = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='same',
name='conv8_2')(net['conv8_1'])
# Last Pool
net['pool6'] = GlobalAveragePooling2D(name='pool6')(net['conv8_2'])
# Prediction from conv4_3
net['conv4_3_norm'] = Normalize(20, name='conv4_3_norm')(
net[prev_size_layer_name])
num_priors = 3
x_out = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='conv4_3_norm_mbox_loc')(net['conv4_3_norm'])
net['conv4_3_norm_mbox_loc'] = x_out
flatten = Flatten(name='conv4_3_norm_mbox_loc_flat')
net['conv4_3_norm_mbox_loc_flat'] = flatten(net['conv4_3_norm_mbox_loc'])
name = 'conv4_3_norm_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x_out = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv4_3_norm'])
net['conv4_3_norm_mbox_conf'] = x_out
flatten = Flatten(name='conv4_3_norm_mbox_conf_flat')
net['conv4_3_norm_mbox_conf_flat'] = flatten(net['conv4_3_norm_mbox_conf'])
priorbox = PriorBox(img_size,
30.0,
aspect_ratios=[2],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv4_3_norm_mbox_priorbox')
net['conv4_3_norm_mbox_priorbox'] = priorbox(net['conv4_3_norm'])
# Prediction from fc7
num_priors = 6
net['fc7_mbox_loc'] = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='fc7_mbox_loc')(net['fc7'])
flatten = Flatten(name='fc7_mbox_loc_flat')
net['fc7_mbox_loc_flat'] = flatten(net['fc7_mbox_loc'])
name = 'fc7_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
net['fc7_mbox_conf'] = Conv2D(num_priors * num_classes, (3, 3),
padding='same',
name=name)(net['fc7'])
flatten = Flatten(name='fc7_mbox_conf_flat')
net['fc7_mbox_conf_flat'] = flatten(net['fc7_mbox_conf'])
priorbox = PriorBox(img_size,
60.0,
max_size=114.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='fc7_mbox_priorbox')
net['fc7_mbox_priorbox'] = priorbox(net['fc7'])
# Prediction from conv6_2
num_priors = 6
x_out = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='conv6_2_mbox_loc')(net['conv6_2'])
net['conv6_2_mbox_loc'] = x_out
flatten = Flatten(name='conv6_2_mbox_loc_flat')
net['conv6_2_mbox_loc_flat'] = flatten(net['conv6_2_mbox_loc'])
name = 'conv6_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x_out = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv6_2'])
net['conv6_2_mbox_conf'] = x_out
flatten = Flatten(name='conv6_2_mbox_conf_flat')
net['conv6_2_mbox_conf_flat'] = flatten(net['conv6_2_mbox_conf'])
priorbox = PriorBox(img_size,
114.0,
max_size=168.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv6_2_mbox_priorbox')
net['conv6_2_mbox_priorbox'] = priorbox(net['conv6_2'])
# Prediction from conv7_2
num_priors = 6
x_out = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='conv7_2_mbox_loc')(net['conv7_2'])
net['conv7_2_mbox_loc'] = x_out
flatten = Flatten(name='conv7_2_mbox_loc_flat')
net['conv7_2_mbox_loc_flat'] = flatten(net['conv7_2_mbox_loc'])
name = 'conv7_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x_out = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv7_2'])
net['conv7_2_mbox_conf'] = x_out
flatten = Flatten(name='conv7_2_mbox_conf_flat')
net['conv7_2_mbox_conf_flat'] = flatten(net['conv7_2_mbox_conf'])
priorbox = PriorBox(img_size,
168.0,
max_size=222.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv7_2_mbox_priorbox')
net['conv7_2_mbox_priorbox'] = priorbox(net['conv7_2'])
# Prediction from conv8_2
num_priors = 6
x_out = Conv2D(num_priors * 4, (3, 3),
padding='same',
name='conv8_2_mbox_loc')(net['conv8_2'])
net['conv8_2_mbox_loc'] = x_out
flatten = Flatten(name='conv8_2_mbox_loc_flat')
net['conv8_2_mbox_loc_flat'] = flatten(net['conv8_2_mbox_loc'])
name = 'conv8_2_mbox_conf'
if num_classes != 21:
name += '_{}'.format(num_classes)
x_out = Conv2D(num_priors * num_classes, (3, 3), padding='same',
name=name)(net['conv8_2'])
net['conv8_2_mbox_conf'] = x_out
flatten = Flatten(name='conv8_2_mbox_conf_flat')
net['conv8_2_mbox_conf_flat'] = flatten(net['conv8_2_mbox_conf'])
priorbox = PriorBox(img_size,
222.0,
max_size=276.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='conv8_2_mbox_priorbox')
net['conv8_2_mbox_priorbox'] = priorbox(net['conv8_2'])
# Prediction from pool6
num_priors = 6
x_out = Dense(num_priors * 4, name='pool6_mbox_loc_flat')(net['pool6'])
net['pool6_mbox_loc_flat'] = x_out
name = 'pool6_mbox_conf_flat'
if num_classes != 21:
name += '_{}'.format(num_classes)
x_out = Dense(num_priors * num_classes, name=name)(net['pool6'])
net['pool6_mbox_conf_flat'] = x_out
priorbox = PriorBox(img_size,
276.0,
max_size=330.0,
aspect_ratios=[2, 3],
variances=[0.1, 0.1, 0.2, 0.2],
name='pool6_mbox_priorbox')
if K.image_dim_ordering() == 'tf':
target_shape = (1, 1, 256)
else:
target_shape = (256, 1, 1)
net['pool6_reshaped'] = Reshape(target_shape,
name='pool6_reshaped')(net['pool6'])
net['pool6_mbox_priorbox'] = priorbox(net['pool6_reshaped'])
# Gather all predictions
net['mbox_loc'] = concatenate([
net['conv4_3_norm_mbox_loc_flat'], net['fc7_mbox_loc_flat'],
net['conv6_2_mbox_loc_flat'], net['conv7_2_mbox_loc_flat'],
net['conv8_2_mbox_loc_flat'], net['pool6_mbox_loc_flat']
],
axis=1,
name='mbox_loc')
net['mbox_conf'] = concatenate([
net['conv4_3_norm_mbox_conf_flat'], net['fc7_mbox_conf_flat'],
net['conv6_2_mbox_conf_flat'], net['conv7_2_mbox_conf_flat'],
net['conv8_2_mbox_conf_flat'], net['pool6_mbox_conf_flat']
],
axis=1,
name='mbox_conf')
net['mbox_priorbox'] = concatenate([
net['conv4_3_norm_mbox_priorbox'], net['fc7_mbox_priorbox'],
net['conv6_2_mbox_priorbox'], net['conv7_2_mbox_priorbox'],
net['conv8_2_mbox_priorbox'], net['pool6_mbox_priorbox']
],
axis=1,
name='mbox_priorbox')
if hasattr(net['mbox_loc'], '_keras_shape'):
num_boxes = net['mbox_loc']._keras_shape[-1] // 4
elif hasattr(net['mbox_loc'], 'int_shape'):
num_boxes = K.int_shape(net['mbox_loc'])[-1] // 4
net['mbox_loc'] = Reshape((num_boxes, 4),
name='mbox_loc_final')(net['mbox_loc'])
net['mbox_conf'] = Reshape((num_boxes, num_classes),
name='mbox_conf_logits')(net['mbox_conf'])
net['mbox_conf'] = Activation('softmax',
name='mbox_conf_final')(net['mbox_conf'])
net['predictions'] = concatenate(
[net['mbox_loc'], net['mbox_conf'], net['mbox_priorbox']],
axis=2,
name='predictions')
model = Model(net['input'], net['predictions'])
model.layers[0].name = 'input_1'
return model
layer2 = Conv2D(filters=64, kernel_size=3, strides=1,
padding='valid')(layer1) # 6
#layer2 = BatchNormalization()(layer2) # 7
layer2 = LeakyReLU(alpha=0.3)(layer2) # 8
layer2 = Dropout(0.2)(layer2) # 9
# print(layer2.get_shape().as_list())
layer3 = Flatten()(layer2) # 10
# add sc input
sc_input = Input(shape=(1, ), dtype='int32')
sc_layer = Embedding(12, 64, input_length=1)(sc_input)
sc_layer = Flatten()(sc_layer)
sc_layer = Dense(70 * 64)(sc_layer)
#sc_layer = BatchNormalization()(sc_layer) # ?
sc_layer = LeakyReLU(alpha=0.3)(sc_layer) # ?
sc_layer = Dropout(0.2)(sc_layer)
layer3 = concatenate([layer3, sc_layer])
layer3 = Dense(7 * 10 * 64)(layer3) # 11
#layer3 = BatchNormalization()(layer3) # 12
layer3 = LeakyReLU(alpha=0.3)(layer3) # 13
layer3 = Dropout(0.2)(layer3) # 14
layer3 = Reshape((7, 10, 64))(layer3) # 15
layer4 = Conv2DTranspose(filters=32,
kernel_size=3,
strides=1,
padding='valid')(layer3) # 16
#layer4 = BatchNormalization()(layer4) # 17
layer4 = LeakyReLU(alpha=0.3)(layer4) # 18
layer4 = Dropout(0.2)(layer4) # 19
predictions = Conv2DTranspose(filters=2,
kernel_size=(4, 5),
strides=2,
def LinkNet(input_shape=(256, 256, 3), classes=2):
inputs = Input(shape=input_shape)
x = BatchNormalization()(inputs)
x = Activation('relu')(x)
# x = lrelu(x)
x = Conv2D(filters=64, kernel_size=(7, 7), strides=(2, 2))(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding="same")(x)
encoder_1 = encoder_block(input_tensor=x, m=64, n=64)
encoder_1 = Dropout(0.2)(encoder_1)
encoder_2 = encoder_block(input_tensor=encoder_1, m=64, n=128)
encoder_2 = Dropout(0.2)(encoder_2)
encoder_3 = encoder_block(input_tensor=encoder_2, m=128, n=256)
encoder_3 = Dropout(0.2)(encoder_3)
encoder_4 = encoder_block(input_tensor=encoder_3, m=256, n=512)
encoder_4 = Dropout(0.2)(encoder_4)
decoder_4 = decoder_block(input_tensor=encoder_4, m=512, n=256)
decoder_4 = Dropout(0.2)(decoder_4)
decoder_3_in = concatenate([decoder_4, encoder_3])
decoder_3_in = Activation('relu')(decoder_3_in)
# decoder_3_in = lrelu(decoder_3_in)
decoder_3 = decoder_block(input_tensor=decoder_3_in, m=256, n=128)
decoder_3 = Dropout(0.2)(decoder_3)
decoder_2_in = concatenate([decoder_3, encoder_2])
decoder_2_in = Activation('relu')(decoder_2_in)
# decoder_2_in = lrelu(decoder_2_in)
decoder_2 = decoder_block(input_tensor=decoder_2_in, m=128, n=64)
decoder_2 = Dropout(0.2)(decoder_2)
decoder_1_in = concatenate([decoder_2, encoder_1])
decoder_1_in = Activation('relu')(decoder_1_in)
# decoder_1_in = lrelu(decoder_1_in)
decoder_1 = decoder_block(input_tensor=decoder_1_in, m=64, n=64)
decoder_1 = Dropout(0.2)(decoder_1)
x = UpSampling2D((2, 2))(decoder_1)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# x = lrelu(x)
x = Conv2D(filters=32, kernel_size=(3, 3), padding="same")(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# x = lrelu(x)
x = Conv2D(filters=32, kernel_size=(3, 3), padding="same")(x)
x = UpSampling2D((2, 2))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# x = lrelu(x)
x = Conv2D(filters=classes, kernel_size=(2, 2), padding="same")(x)
output = Reshape([-1, classes])(x)
output = Activation('softmax')(output)
x = Reshape(input_shape[:-1] + (classes, ))(output)
model = Model(inputs=inputs, outputs=x)
return model
yAv = layers.Reshape(in_shp_yAv + (1, ))(input_yAv)
v_in = layers.Reshape(in_shp_v + (1, ))(input_v)
for _ in range(3):
yAv = layers.Conv1D(rows,
3,
data_format="channels_first",
activation='relu',
padding='same')(yAv)
for _ in range(3):
yAv = layers.Conv1D(cols,
3,
data_format="channels_last",
activation='relu',
padding='same')(yAv)
yAv2D = layers.Reshape((rows, cols, 1))(yAv)
H_stack = layers.concatenate([yAv2D, v_in], axis=-1)
H = layers.Conv2D(n_channels, (3, 3), activation='relu',
padding='same')(H_stack)
for _ in range(5):
H = layers.Conv2D(n_channels, (3, 3), activation='relu', padding='same')(H)
H_tanh = layers.Conv2D(1, (3, 3), activation='tanh', padding='same')(H)
H_out = layers.Flatten()(H_tanh)
model = models.Model(inputs=[input_yAv, input_v], output=H_out)
model = multi_gpu_model(model, gpus=3)
model.summary()
###############
# Start training
batch_size = 256
model.compile(loss='mean_squared_error', optimizer=Adam(lr=0.0002))
def create_model(self):
'''
Modify the generator and discriminator moodel architecture here.
'''
## generator architecture
init_img_width = self.img_width // 4
init_img_height = self.img_height // 4
random_input = Input(shape=(self.random_input_dim, ))
text_input1 = Input(shape=(self.text_input_dim, ))
random_dense = Dense(1024)(random_input)
text_layer1 = Dense(1024)(text_input1)
merged = concatenate([random_dense, text_layer1])
generator_layer = LeakyReLU()(merged)
generator_layer = Dense(256 * init_img_width *
init_img_height)(generator_layer)
generator_layer = BatchNormalization()(generator_layer)
generator_layer = LeakyReLU()(generator_layer)
generator_layer = Reshape(
(init_img_width, init_img_height, 256), )(generator_layer)
# Deconvolution blocks for generator
#block 1
generator_layer = Conv2DTranspose(128, (5, 5),
strides=(1, 1),
padding='same',
use_bias=False)(generator_layer)
generator_layer = BatchNormalization()(generator_layer)
generator_layer = LeakyReLU()(generator_layer)
#block 2
generator_layer = Conv2DTranspose(64, (5, 5),
strides=(2, 2),
padding='same',
use_bias=False)(generator_layer)
generator_layer = BatchNormalization()(generator_layer)
generator_layer = LeakyReLU()(generator_layer)
#block 3
generator_layer = Conv2DTranspose(3, (5, 5),
strides=(2, 2),
padding='same',
use_bias=False)(generator_layer)
generator_layer = BatchNormalization()(generator_layer)
generator_output = Activation('tanh')(generator_layer)
self.generator = Model([random_input, text_input1], generator_output)
learning_rate = 0.005
g_optim = SGD(lr=learning_rate, momentum=0.9, nesterov=True)
self.generator.compile(loss='binary_crossentropy', optimizer=g_optim)
with open('generator_arch_' + dt_string + '.txt', 'w') as f:
self.generator.summary(print_fn=lambda x: f.write(x + '\n'))
text_input2 = Input(shape=(self.text_input_dim, ))
text_layer2 = Dense(1024)(text_input2)
img_input2 = Input(shape=(self.img_width, self.img_height,
self.img_channels))
## discriminator architecture
# Convolution blocks for discriminator
#### block 1
img_layer2 = Conv2D(64, kernel_size=5, strides=(2, 2),
padding='same')(img_input2)
img_layer2 = LeakyReLU()(img_layer2)
img_layer2 = MaxPooling2D(pool_size=(2, 2))(img_layer2)
#### block 2
img_layer2 = Conv2D(128,
kernel_size=(5, 5),
strides=(2, 2),
padding='same')(img_layer2)
img_layer2 = LeakyReLU()(img_layer2)
img_layer2 = MaxPooling2D(pool_size=(2, 2))(img_layer2)
img_layer2 = Flatten()(img_layer2)
img_layer2 = Dense(1024)(img_layer2)
merged = concatenate([img_layer2, text_layer2])
discriminator_layer = Activation('tanh')(merged)
discriminator_layer = Dense(1)(discriminator_layer)
discriminator_output = Activation('sigmoid')(discriminator_layer)
self.discriminator = Model([img_input2, text_input2],
discriminator_output)
d_optim = SGD(lr=learning_rate, momentum=0.9, nesterov=True)
self.discriminator.compile(loss='binary_crossentropy',
optimizer=d_optim)
with open('discriminator_arch_' + dt_string + '.txt', 'w') as f:
self.discriminator.summary(print_fn=lambda x: f.write(x + '\n'))
model_output = self.discriminator([self.generator.output, text_input1])
self.model = Model([random_input, text_input1], model_output)
self.discriminator.trainable = False
self.model.compile(loss='binary_crossentropy', optimizer=g_optim)
def unet_model(n_classes=5,
im_sz=160,
n_channels=8,
n_filters_start=32,
growth_factor=2,
upconv=True,
class_weights=[0.2, 0.3, 0.1, 0.1, 0.3]):
droprate = 0.25
n_filters = n_filters_start
inputs = Input((im_sz, im_sz, n_channels))
#inputs = BatchNormalization()(inputs)
conv1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(inputs)
conv1 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
#pool1 = Dropout(droprate)(pool1)
n_filters *= growth_factor
pool1 = BatchNormalization()(pool1)
conv2 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(pool1)
conv2 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
pool2 = Dropout(droprate)(pool2)
n_filters *= growth_factor
pool2 = BatchNormalization()(pool2)
conv3 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(pool2)
conv3 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
pool3 = Dropout(droprate)(pool3)
n_filters *= growth_factor
pool3 = BatchNormalization()(pool3)
conv4_0 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(pool3)
conv4_0 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv4_0)
pool4_1 = MaxPooling2D(pool_size=(2, 2))(conv4_0)
pool4_1 = Dropout(droprate)(pool4_1)
n_filters *= growth_factor
pool4_1 = BatchNormalization()(pool4_1)
conv4_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(pool4_1)
conv4_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv4_1)
pool4_2 = MaxPooling2D(pool_size=(2, 2))(conv4_1)
pool4_2 = Dropout(droprate)(pool4_2)
n_filters *= growth_factor
conv5 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(pool4_2)
conv5 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv5)
n_filters //= growth_factor
if upconv:
up6_1 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv5), conv4_1
])
else:
up6_1 = concatenate([UpSampling2D(size=(2, 2))(conv5), conv4_1])
up6_1 = BatchNormalization()(up6_1)
conv6_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(up6_1)
conv6_1 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv6_1)
conv6_1 = Dropout(droprate)(conv6_1)
n_filters //= growth_factor
if upconv:
up6_2 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv6_1), conv4_0
])
else:
up6_2 = concatenate([UpSampling2D(size=(2, 2))(conv6_1), conv4_0])
up6_2 = BatchNormalization()(up6_2)
conv6_2 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(up6_2)
conv6_2 = Conv2D(n_filters, (3, 3), activation='relu',
padding='same')(conv6_2)
conv6_2 = Dropout(droprate)(conv6_2)
n_filters //= growth_factor
if upconv:
up7 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv6_2), conv3
])
else:
up7 = concatenate([UpSampling2D(size=(2, 2))(conv6_2), conv3])
up7 = BatchNormalization()(up7)
conv7 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(up7)
conv7 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv7)
conv7 = Dropout(droprate)(conv7)
n_filters //= growth_factor
if upconv:
up8 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv7), conv2
])
else:
up8 = concatenate([UpSampling2D(size=(2, 2))(conv7), conv2])
up8 = BatchNormalization()(up8)
conv8 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(up8)
conv8 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv8)
conv8 = Dropout(droprate)(conv8)
n_filters //= growth_factor
if upconv:
up9 = concatenate([
Conv2DTranspose(n_filters, (2, 2), strides=(2, 2),
padding='same')(conv8), conv1
])
else:
up9 = concatenate([UpSampling2D(size=(2, 2))(conv8), conv1])
conv9 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(up9)
conv9 = Conv2D(n_filters, (3, 3), activation='relu', padding='same')(conv9)
conv10 = Conv2D(n_classes, (1, 1), activation='sigmoid')(conv9)
model = Model(inputs=inputs, outputs=conv10)
def weighted_binary_crossentropy(y_true, y_pred):
class_loglosses = K.mean(K.binary_crossentropy(y_true, y_pred),
axis=[0, 1, 2])
return K.sum(class_loglosses * K.constant(class_weights))
model.compile(optimizer=Adam(),
loss=weighted_binary_crossentropy,
metrics=['acc'])
return model
embed_char_out = TimeDistributed(Embedding(
len(char2Idx),
30,
embeddings_initializer=RandomUniform(minval=-0.5, maxval=0.5)),
name='char_embedding')(character_input)
dropout = Dropout(0.5)(embed_char_out)
conv1d_out = TimeDistributed(
Conv1D(kernel_size=3,
filters=30,
padding='same',
activation='tanh',
strides=1))(dropout)
maxpool_out = TimeDistributed(MaxPooling1D(52))(conv1d_out)
char = TimeDistributed(Flatten())(maxpool_out)
char = Dropout(0.5)(char)
output = concatenate([words, casing, char])
output = Bidirectional(
LSTM(200, return_sequences=True, dropout=0.50,
recurrent_dropout=0.25))(output)
output = TimeDistributed(Dense(len(label2Idx), activation='softmax'))(output)
model = Model(inputs=[words_input, casing_input, character_input],
outputs=[output])
model.compile(loss='sparse_categorical_crossentropy', optimizer='nadam')
model.summary()
# plot_model(model, to_file='model.png')
for epoch in range(epochs):
print("Epoch %d/%d" % (epoch, epochs))
a = Progbar(len(train_batch_len))
for i, batch in enumerate(iterate_minibatches(train_batch,
train_batch_len)):
pos1_input = Input(shape=(FIXED_SIZE, ), dtype="int32")
pos2_input = Input(shape=(FIXED_SIZE, ), dtype="int32")
# e1_pos = Input(shape=(1,), dtype="int32")
# e2_pos = Input(shape=(1,), dtype="int32")
pos1_embedding = Embedding(input_dim=2 * FIXED_SIZE - 1, output_dim=POS_EMBEDDING_DIM, input_length=FIXED_SIZE,
trainable=True) \
(pos1_input)
pos2_embedding = Embedding(input_dim=2 * FIXED_SIZE - 1, output_dim=POS_EMBEDDING_DIM, input_length=FIXED_SIZE,
trainable=True) \
(pos2_input)
pos_embedding = concatenate([pos1_embedding, pos2_embedding], axis=2)
word_embedding = Embedding(input_dim=len(vec),
output_dim=EMBEDDING_DIM,
weights=[vec],
input_length=FIXED_SIZE,
trainable=False)(index_input)
embedding_output = concatenate([word_embedding, pos_embedding], axis=2)
# embedding_output = add([word_embedding, pos_embedding])
cnn1 = Conv1D(filters=150,
kernel_size=2,
strides=1,
padding="same",
activation="relu")(embedding_output)
def get_unet(img_rows, img_cols, loss , optimizer, metrics, channels = 1,num_class=1):
# attention unet with dilated conv
inputs = Input((img_rows, img_cols, channels))
num_f = [4,8,16,32,64,128,256,512,1024] # numbers of filters
# gaussian_noise_std = 0.025
# input_with_noise = GaussianNoise(gaussian_noise_std)(inputs)
conv1 = Conv2D(num_f[0], (3, 3), activation='relu', padding='same')(inputs)
conv1 = Conv2D(num_f[0], (3, 3), activation='relu', padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(num_f[1], (3, 3), activation='relu', padding='same')(pool1)
conv2 = Conv2D(num_f[1], (3, 3), activation='relu', padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(num_f[2], (3, 3), activation='relu', padding='same')(pool2)
conv3 = Conv2D(num_f[2], (3, 3), activation='relu', padding='same')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(num_f[3], (3, 3), activation='relu', padding='same')(pool3)
conv4 = Conv2D(num_f[3], (3, 3), activation='relu', padding='same')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = Conv2D(num_f[4], (3, 3), activation='relu', padding='same')(pool4)
# dilated conv with rate 1,2,5,1,2,5
conv5 = Conv2D(num_f[4], (3, 3), activation='relu', padding='same')(conv5)
conv5 = Conv2D(num_f[4], (3, 3), strides=(1,1), dilation_rate=2, activation='relu', padding='same')(conv5)
conv5 = Conv2D(num_f[4], (3, 3), strides=(1,1), dilation_rate=5, activation='relu', padding='same')(conv5)
conv5 = Conv2D(num_f[4], (3, 3), activation='relu', padding='same')(conv5)
conv5 = Conv2D(num_f[4], (3, 3), strides=(1,1), dilation_rate=2, activation='relu', padding='same')(conv5)
conv5 = Conv2D(num_f[4], (3, 3), strides=(1,1), dilation_rate=5, activation='relu', padding='same')(conv5)
up6 = Conv2DTranspose(num_f[3], (2, 2), strides=(2, 2), padding='same')(conv5)
at6 = Attention_bolck(up6, conv4)
up6 = concatenate([up6,at6], axis=3)
conv6 = Conv2D(num_f[3], (3, 3), activation='relu', padding='same')(up6)
conv6 = Conv2D(num_f[3], (3, 3), activation='relu', padding='same')(conv6)
up7 = Conv2DTranspose(num_f[2], (2, 2), strides=(2, 2), padding='same')(conv6)
at7 = Attention_bolck(up7, conv3)
up7 = concatenate([up7, at7], axis=3)
conv7 = Conv2D(num_f[2], (3, 3), activation='relu', padding='same')(up7)
conv7 = Conv2D(num_f[2], (3, 3), activation='relu', padding='same')(conv7)
up8 = Conv2DTranspose(num_f[1], (2, 2), strides=(2, 2), padding='same')(conv7)
at8 = Attention_bolck(up8,conv2)
up8 = concatenate([up8, at8], axis=3)
conv8 = Conv2D(num_f[1], (3, 3), activation='relu', padding='same')(up8)
conv8 = Conv2D(num_f[1], (3, 3), activation='relu', padding='same')(conv8)
up9 = Conv2DTranspose(num_f[0], (2, 2), strides=(2, 2), padding='same')(conv8)
at9 = Attention_bolck(up9,conv1)
up9 = concatenate([up9, at9], axis=3)
conv9 = Conv2D(num_f[0], (3, 3), activation='relu', padding='same')(up9)
conv9 = Conv2D(num_f[0], (3, 3), activation='relu', padding='same')(conv9)
conv10 = Conv2D(num_class, (1, 1), activation='sigmoid',name='c')(conv9) #未考虑背景不能用softmax
model = Model(inputs=[inputs], outputs=[conv10])
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
return model
padding='same',
strides=1)(input)
if __name__ == '__main__':
feature_sizes = [8, 4, 16, 16, 8, 12, 16, 12, 10]
inputs = [Input((1000, size)) for size in feature_sizes]
convs = [
inputs[item] if item == 2 else conv_block_for_features(
16, 2, inputs[item]) for item in range(8)
]
convs_concat = concatenate(convs)
batch_norm = BatchNormalization()(convs_concat)
conv1 = common_conv_skip_block(64, 2, batch_norm)
conv2 = common_conv_skip_block(64, 3, batch_norm)
conv3 = common_conv_skip_block(64, 4, batch_norm)
output_3 = concatenate([conv1, conv2, conv3])
batch_norm_2 = BatchNormalization()(output_3)
bilstm = Bidirectional(LSTM(units=100,
return_sequences=True))(batch_norm_2)
gb_maxpool = GlobalMaxPooling1D()(bilstm)
dense_1 = Dense(units=128, kernel_initializer='uniform',
activation='relu')(gb_maxpool)
