def my_assessor_model(mouth_nn='cnn', mouth_features_dim=512, lstm_units_1=32, dense_fc_1=128, dense_fc_2=64,
conv_f_1=32, conv_f_2=64, conv_f_3=128, dropout_p=0.2):
mouth_input_shape = (MOUTH_H, MOUTH_W, MOUTH_CHANNELS)
my_input_mouth_images = Input(shape=(TIME_STEPS, *mouth_input_shape))
my_input_head_poses = Input(shape=(TIME_STEPS, 3))
my_input_n_of_frames = Input(shape=(1,))
my_input_lipreader_dense = Input(shape=(1024,))
my_input_lipreader_softmax = Input(shape=(500,))
if mouth_nn == 'cnn':
mouth_feature_model = my_timedistributed_cnn_model((TIME_STEPS, *mouth_input_shape), conv_f_1, conv_f_2, conv_f_3, mouth_features_dim)
elif mouth_nn == 'resnet18':
mouth_feature_model = ResnetBuilder.build_resnet_18((TIME_STEPS, *mouth_input_shape), mouth_features_dim, time_distributed=True)
elif mouth_nn == 'resnet34':
mouth_feature_model = ResnetBuilder.build_resnet_34((TIME_STEPS, *mouth_input_shape), mouth_features_dim, time_distributed=True)
elif mouth_nn == 'resnet50':
mouth_feature_model = ResnetBuilder.build_resnet_50((TIME_STEPS, *mouth_input_shape), mouth_features_dim, time_distributed=True)
elif mouth_nn == 'resnet101':
mouth_feature_model = ResnetBuilder.build_resnet_101((TIME_STEPS, *mouth_input_shape), mouth_features_dim, time_distributed=True)
elif mouth_nn == 'resnet152':
mouth_feature_model = ResnetBuilder.build_resnet_152((TIME_STEPS, *mouth_input_shape), mouth_features_dim, time_distributed=True)
elif mouth_nn == 'flatten':
mouth_feature_model = Reshape((TIME_STEPS, -1), input_shape=(TIME_STEPS, *mouth_input_shape))
cnn_features = mouth_feature_model(my_input_mouth_images)
lstm_input = Concatenate()([cnn_features, my_input_head_poses])
lstm_output = LSTM(lstm_units_1, activation='tanh', kernel_regularizer=l2(1.e-4), return_sequences=False)(lstm_input)
concatenated_features = Concatenate()([lstm_output, my_input_n_of_frames, my_input_lipreader_dense, my_input_lipreader_softmax])
fc1 = Dense(dense_fc_1, activation='relu', kernel_regularizer=l2(1.e-4))(concatenated_features)
bn1 = BatchNormalization()(fc1)
dp1 = Dropout(dropout_p)(bn1)
fc2 = Dense(dense_fc_2, activation='relu', kernel_regularizer=l2(1.e-4))(dp1)
bn2 = BatchNormalization()(fc2)
dp2 = Dropout(dropout_p)(bn2)
assessor_output = Dense(1, activation='sigmoid')(dp2)
assessor = Model(inputs=[my_input_mouth_images, my_input_head_poses, my_input_n_of_frames, my_input_lipreader_dense, my_input_lipreader_softmax],
outputs=assessor_output)
assessor.summary()
return assessor
def get_model(model, ishape, nb_classes):
"Return proper ResNet model"
if model.endswith('18'):
model = ResnetBuilder.build_resnet_18(ishape, nb_classes)
elif model.endswith('34'):
model = ResnetBuilder.build_resnet_34(ishape, nb_classes)
elif model.endswith('50'):
model = ResnetBuilder.build_resnet_50(ishape, nb_classes)
elif model.endswith('101'):
model = ResnetBuilder.build_resnet_101(ishape, nb_classes)
elif model.endswith('152'):
model = ResnetBuilder.build_resnet_152(ishape, nb_classes)
else:
return NotImplemented()
return model
def model_confirm(self, choosed_model):
if choosed_model == 'AlexNet':
model = alexnet(self.config)
elif choosed_model == 'VGG16':
model = vgg16(self.config)
elif choosed_model == 'VGG19':
model = vgg19(self.config)
elif choosed_model == 'InceptionV3':
model = inception_v3(self.config)
elif choosed_model == 'InceptionV4':
model = inception_v4(self.config)
elif choosed_model == 'InceptionV4_ResNetV1':
model = inception_resnet_v1(self.config)
elif choosed_model == 'InceptionV4_ResNetV2':
model = inception_resnet_v2(self.config)
elif choosed_model == 'ResNet18':
model = ResnetBuilder.build_resnet_18(self.config)
elif choosed_model == 'ResNet34':
model = ResnetBuilder.build_resnet_34(self.config)
elif choosed_model == 'ResNet50':
model = ResnetBuilder.build_resnet_50(self.config)
elif choosed_model == 'ResNet101':
model = ResnetBuilder.build_resnet_101(self.config)
elif choosed_model == 'ResNet152':
model = ResnetBuilder.build_resnet_152(self.config)
elif choosed_model == 'DenseNet121':
model = densenet121(self.config)
elif choosed_model == 'DenseNet169':
model = densenet169(self.config)
elif choosed_model == 'DenseNet201':
model = densenet201(self.config)
elif choosed_model == 'DenseNet264':
model = densenet264(self.config)
else:
model = -1
return model
def create_model(self, model_type='xception', load_weights=None):
if (model_type == 'inceptionv3' or model_type == 1):
base = InceptionV3(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'inceptionv3'
pred = base.output
elif (model_type == 'resnet50' or model_type == 2):
base = ResNet50(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'resnet50'
pred = base.output
elif (model_type == 'vgg19' or model_type == 3):
base = VGG19(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'vgg19'
pred = base.output
elif (model_type == 'vgg16' or model_type == 4):
base = VGG16(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'vgg16'
pred = base.output
elif (model_type == 'resnet152' or model_type == 5):
resbuild = ResnetBuilder()
base = resbuild.build_resnet_152(self.input_shape,
self.output_size)
model_name = 'resnet152'
pred = base.output
elif (model_type == 'resnet50MOD' or model_type == 6):
resbuild = ResnetBuilder()
base = resbuild.build_resnet_50(self.input_shape, self.output_size)
model_name = 'resnet50MOD'
pred = base.output
elif (model_type == 'inceptionv3MOD' or model_type == 7):
base = InceptionV3MOD(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'inceptionv3MOD'
pred = base.output
else:
base = Xception(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'xception'
pred = base.output
pred = Dense(self.output_size,
activation='sigmoid',
name='predictions')(pred)
self.model = Model(base.input, pred, name=model_name)
if load_weights != None:
self.model.load_weights(load_weights)
for layer in base.layers:
layer.trainable = True
self.model.compile(loss=losses.binary_crossentropy,
optimizer='adam',
metrics=[FScore2])
def test_custom2():
""" https://github.com/raghakot/keras-resnet/issues/34
"""
model = ResnetBuilder.build_resnet_152((3, 512, 512), 2)
_test_model_compile(model)
def test_resnet152():
model = ResnetBuilder.build_resnet_152((3, 224, 224), 100)
_test_model_compile(model)
def test_resnet152():
for ordering in DIM_ORDERING:
K.set_image_dim_ordering(ordering)
model = ResnetBuilder.build_resnet_152((3, 224, 224), 100)
model.compile(loss="categorical_crossentropy", optimizer="sgd")
assert True, "Failed to build with '{}' dim ordering".format(ordering)
def build_resnet_152(self, shape, num_classes):
self.model = ResnetBuilder.build_resnet_152(shape, num_classes)
print('[+] ResNet-152 model built')
return self.model
def create_model(self, model_type='xception', load_weights=None):
if (model_type == 'inceptionv3' or model_type == 1):
base = InceptionV3(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'inceptionv3'
pred = base.output
elif (model_type == 'resnet50' or model_type == 2):
base = ResNet50(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'resnet50'
pred = base.output
elif (model_type == 'vgg19' or model_type == 3):
base = VGG19(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'vgg19'
pred = base.output
elif (model_type == 'vgg16' or model_type == 4):
base = VGG16(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'vgg16'
pred = base.output
elif (model_type == 'resnet152' or model_type == 5):
sys.path.append(os.path.join(PATH, "resnet", "keras-resnet"))
from resnet import ResnetBuilder
resbuild = ResnetBuilder()
base = resbuild.build_resnet_152(self.input_shape,
self.output_size)
model_name = 'resnet152'
pred = base.output
elif (model_type == 'resnet50MOD' or model_type == 6):
sys.path.append(os.path.join(PATH, "resnet", "keras-resnet"))
from resnet import ResnetBuilder
resbuild = ResnetBuilder()
base = resbuild.build_resnet_50(self.input_shape, self.output_size)
model_name = 'resnet50MOD'
pred = base.output
elif (model_type == 'inceptionv3MOD' or model_type == 7):
from keras.applications.inception_v3_mod import InceptionV3MOD
base = InceptionV3MOD(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'inceptionv3MOD'
pred = base.output
else:
base = Xception(include_top=False,
weights='imagenet',
input_tensor=self.input_tensor,
classes=self.output_size,
pooling='avg')
model_name = 'xception'
pred = base.output
pred = Dense(self.output_size,
activation='sigmoid',
name='predictions')(pred)
self.model = Model(base.input, pred, name=model_name)
if load_weights != None:
self.model.load_weights(load_weights)
for layer in base.layers:
layer.trainable = True
self.compile()
def test_resnet152():
model = ResnetBuilder.build_resnet_152((1, 200, 200), 100)
_test_model_compile(model)
def test_custom2():
""" https://github.com/raghakot/keras-resnet/issues/34
"""
model = ResnetBuilder.build_resnet_152((3, 512, 512), 2)
_test_model_compile(model)
def test_resnet152():
model = ResnetBuilder.build_resnet_152((3, 224, 224), 100)
_test_model_compile(model)
