def create_MobileNet_with_drop(self, inp_shape, inp_tensor, output_len,
weight_path):
initializer = tf.keras.initializers.glorot_uniform()
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=inp_tensor,
pooling=None)
mobilenet_model.layers.pop()
x = mobilenet_model.get_layer(
'global_average_pooling2d').output # 1280
x = Dense(output_len, name='O_L')(x)
inp = mobilenet_model.input
model = Model(inp, x)
model.load_weights(weight_path)
model.summary()
'''revise model and add droput'''
model.layers.pop()
x = model.get_layer('global_average_pooling2d').output # 1280
x = Dropout(0.5)(x)
out_landmarks = Dense(output_len,
activation=keras.activations.linear,
use_bias=False,
kernel_initializer=initializer,
name='O_L')(x)
inp = mobilenet_model.input
revised_model = Model(inp, out_landmarks)
revised_model.summary()
revised_model.save_weights('W_ds_wflw_mn_base_with_drop.h5')
revised_model.save('M_ds_wflw_mn_base_with_drop.h5')
model_json = revised_model.to_json()
with open("mobileNet_v2_main.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def mobileNet_v2_main_discriminator(self, tensor, input_shape):
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=tensor,
pooling=None)
# , classes=cnf.landmark_len)
mobilenet_model.layers.pop()
x = mobilenet_model.get_layer(
'global_average_pooling2d_2').output # 1280
softmax = Dense(1, activation='sigmoid', name='out')(x)
inp = mobilenet_model.input
revised_model = Model(inp, softmax)
revised_model.summary()
# plot_model(revised_model, to_file='mobileNet_v2_main.png', show_shapes=True, show_layer_names=True)
model_json = revised_model.to_json()
with open("mobileNet_v2_main.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def mobileNet_v2_main(self, tensor):
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=[InputDataSize.image_input_size, InputDataSize.image_input_size,3],
alpha=1.0,
include_top=True,
weights=None,
input_tensor=tensor,
pooling=None)
mobilenet_model.summary()
# , classes=cnf.landmark_len)
mobilenet_model.layers.pop()
x = mobilenet_model.get_layer('global_average_pooling2d').output # 1280
x = Dense(136, name='dense_layer_out_2', activation='relu',
kernel_initializer='he_uniform')(x)
Logits = Dense(136, name='out')(x)
inp = mobilenet_model.input
revised_model = Model(inp, Logits)
revised_model.summary()
# plot_model(revised_model, to_file='mobileNet_v2_main.png', show_shapes=True, show_layer_names=True)
model_json = revised_model.to_json()
with open("mobileNet_v2_main.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def mnv2_hm(self, tensor):
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=None,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=tensor,
pooling=None)
mobilenet_model.layers.pop()
inp = mobilenet_model.input
'''heatmap can not be generated from activation layers, so we use out_relu'''
x = mobilenet_model.get_layer('out_relu').output # 7, 7, 1280
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv1',
kernel_initializer='he_uniform')(x) # 14, 14, 256
x = BatchNormalization(name='out_bn1')(x)
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv2',
kernel_initializer='he_uniform')(x) # 28, 28, 256
x = BatchNormalization(name='out_bn2')(x)
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv3',
kernel_initializer='he_uniform')(x) # 56, 56, 256
x = BatchNormalization(name='out_bn3')(x)
out_heatmap = Conv2D(LearningConfig.landmark_len // 2,
kernel_size=1,
padding='same',
name='out_heatmap')(x)
revised_model = Model(inp, [
out_heatmap,
])
revised_model.summary()
# plot_model(revised_model, to_file='mnv2_hm.png', show_shapes=True, show_layer_names=True)
model_json = revised_model.to_json()
with open("mnv2_hm.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def __init__(self, model_name='resnet'):
if (model_name == 'mobilenet'):
print("Using MobileNetV2 to vectorize images")
model = mobilenet_v2.MobileNetV2(weights='imagenet')
layer_name = 'global_average_pooling2d'
self.preprocess_fn = mobilenet_v2.preprocess_input
else:
print("Using ResNet50 to vectorize images")
model = resnet50.ResNet50(weights='imagenet')
layer_name = 'avg_pool'
self.preprocess_fn = resnet50.preprocess_input
o = model.get_layer(layer_name).output
self.vec_len = o.shape[1]
self.intermediate_layer_model = Model(inputs=model.input,
outputs=o)
def create_MobileNet(self, inp_shape, inp_tensor):
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=inp_tensor,
pooling=None)
inp = mobilenet_model.input
out_landmarks = mobilenet_model.get_layer('O_L').output
revised_model = Model(inp, [out_landmarks])
model_json = revised_model.to_json()
with open("mobileNet_v2_stu.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def create_branch_mn(self, prefix, inp_shape):
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model.layers.pop()
inp = mobilenet_model.input
'''heatmap can not be generated from activation layers, so we use out_relu'''
x = mobilenet_model.get_layer('out_relu').output # 7, 7, 1280
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name=prefix + '_deconv1',
kernel_initializer='he_uniform')(x) # 14, 14, 256
x = BatchNormalization(name=prefix + 'out_bn1')(x)
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name=prefix + '_deconv2',
kernel_initializer='he_uniform')(x) # 28, 28, 256
x = BatchNormalization(name=prefix + 'out_bn2')(x)
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name=prefix + '_deconv3',
kernel_initializer='he_uniform')(x) # 56, 56, 256
x = BatchNormalization(name=prefix + 'out_bn3')(x)
out_heatmap = Conv2D(LearningConfig.point_len,
kernel_size=1,
padding='same',
name=prefix + '_out_hm')(x)
for layer in mobilenet_model.layers:
layer.name = layer.name + '_' + prefix
return inp, out_heatmap
def create_MobileNet_dif(self, inp_shape, inp_tensor, output_len, is_old,
weight_path):
initializer = tf.keras.initializers.glorot_uniform()
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=inp_tensor,
pooling=None)
mobilenet_model.layers.pop()
# mobilenet_model.summary()
# global_avg = mobilenet_model.get_layer('global_average_pooling2d_2').output # 1280
global_avg = mobilenet_model.get_layer(
'global_average_pooling2d').output # 1280
x = Dense(3 * output_len)(global_avg)
x = BatchNormalization()(x)
x = ReLU()(x)
x = Dense(2 * output_len)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = Dropout(0.3)(x)
dif_gt_st = Dense(output_len, name='dif_gt_st')(x)
x = Dense(3 * output_len)(global_avg)
x = BatchNormalization()(x)
x = ReLU()(x)
x = Dense(2 * output_len)(x)
x = BatchNormalization()(x)
x = ReLU()(x)
x = Dropout(0.3)(x)
dif_gt_pt = Dense(output_len, name='dif_gt_pt')(x)
'''now we add other layers'''
# global_avg = revised_model.get_layer('global_average_pooling2d').output # 1280
inp = mobilenet_model.input
revised_model = Model(inp, [dif_gt_st, dif_gt_pt])
model_json = revised_model.to_json()
# revised_model.save('ds_300w_stu_.h5')
with open("mobileNet_v2_stu_dif.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def create_MobileNet(self, inp_shape, inp_tensor, output_len, is_old,
weight_path):
# initializer = tf.keras.initializers.HeUniform()
initializer = tf.keras.initializers.glorot_uniform()
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=inp_tensor,
pooling=None)
# mobilenet_model.summary()
mobilenet_model.layers.pop()
x = mobilenet_model.get_layer(
'global_average_pooling2d_1').output # 1280
# x = mobilenet_model.get_layer('global_average_pooling2d').output # 1280
# x = Dropout(0.5)(x)
out_landmarks = Dense(output_len, name='O_L')(x)
inp = mobilenet_model.input
revised_model = Model(inp, out_landmarks)
revised_model.summary()
if is_old:
revised_model.load_weights(weight_path)
'''now we add other layers'''
# global_avg = revised_model.get_layer('global_average_pooling2d').output # 1280
inp = revised_model.input
out_landmarks = revised_model.get_layer('O_L').output
revised_model = Model(inp, [out_landmarks])
model_json = revised_model.to_json()
# revised_model.save('ds_300w_stu_.h5')
with open("mobileNet_v2_stu.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def create_asmnet(self, inp_shape, num_branches, output_len):
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model.layers.pop()
inp = mobilenet_model.input
outputs = []
relu_name = 'out_relu'
for i in range(num_branches):
x = mobilenet_model.get_layer(relu_name).output # 7, 7, 1280
prefix = str(i)
for layer in mobilenet_model.layers:
layer.name = layer.name + prefix
relu_name = relu_name + prefix
'''heatmap can not be generated from activation layers, so we use out_relu'''
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name=prefix + '_deconv1',
kernel_initializer='he_uniform')(
x) # 14, 14, 256
x = BatchNormalization(name=prefix + 'out_bn1')(x)
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name=prefix + '_deconv2',
kernel_initializer='he_uniform')(
x) # 28, 28, 256
x = BatchNormalization(name=prefix + 'out_bn2')(x)
x = Deconvolution2D(filters=256,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name=prefix + '_deconv3',
kernel_initializer='he_uniform')(
x) # 56, 56, 256
x = BatchNormalization(name=prefix + 'out_bn3')(x)
out_heatmap = Conv2D(output_len,
kernel_size=1,
padding='same',
name=prefix + '_out_hm')(x)
outputs.append(out_heatmap)
revised_model = Model(inp, outputs)
revised_model.summary()
model_json = revised_model.to_json()
with open("asmnet.json", "w") as json_file:
json_file.write(model_json)
return revised_model
# process an image to be model friendly
def process_image(img_path):
img = image.load_img(img_path, target_size=(224, 224))
img_array = image.img_to_array(img)
img_array = np.expand_dims(img_array, axis=0)
pImg = mobilenet_v2.preprocess_input(img_array)
return pImg
test_img_path = '%s/Abyssinian_1.jpg' % images_dir
pImg = process_image(test_img_path)
# define the model
model = mobilenet_v2.MobileNetV2(input_shape=None,
alpha=1.0,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000)
# display top %activation_top_layers% activations
top_layers = 0
layers_outputs = [
layer.output for layer in model.layers
if (layer.__class__.__name__ == 'Conv2D')
][top_layers:]
layers_names = [layer.name for layer in layers_outputs]
activation_model = Model(inputs=model.input, outputs=layers_outputs)
activation_model.summary()
target_size=image_size,
batch_size=batch_size,
shuffle=True,
# subset='training'
)
validation_generator = train_datagen.flow_from_directory(
validation_path,
target_size=image_size,
batch_size=batch_size,
shuffle=True,
# subset='validation'
)
conv_layers = mobilenet_v2.MobileNetV2(weights='imagenet',
include_top=False,
input_shape=(image_size[0],
image_size[1], 3))
model = tf.keras.models.Sequential()
model.add(conv_layers)
model.add(layers.Flatten())
model.add(layers.Dense(1024, activation='relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dense(n_class, activation='softmax'))
print(model.summary())
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = model.fit(train_generator,
def __call__(self, *args, **kwargs):
if self.model_name in ["VGG16", "vgg16"]:
pre_trained = vgg16.VGG16(include_top=False,
weights='imagenet',
input_shape=self.input_shape)
elif self.model_name in ["VGG19", "vgg19"]:
pre_trained = vgg19.VGG19(include_top=False,
weights='imagenet',
input_shape=self.input_shape)
elif self.model_name in ["MobileNet", "mobilenet"]:
pre_trained = mobilenet.MobileNet(include_top=False,
weights='imagenet',
input_shape=self.input_shape)
elif self.model_name in [
"MobileNetV2", "MobileNet_V2", "mobilenetv2", "mobilenet_v2",
"mobilenet_V2"
]:
pre_trained = mobilenet_v2.MobileNetV2(
include_top=False,
weights='imagenet',
input_shape=self.input_shape)
elif self.model_name in ["resnet50", "ResNet50"]:
pre_trained = resnet.ResNet50(include_top=False,
weights='imagenet',
input_shape=self.input_shape)
# elif self.model_name in ["EfficientNetB0", "efficientnetb0"]:
# pre_trained = efficientnet.EfficientNetB0(include_top=False, weights='imagenet', input_shape=self.input_shape)
#
# elif self.model_name in ["EfficientNetB5", "efficientnetb5"]:
# pre_trained = efficientnet.EfficientNetB5(include_top=False, weights='imagenet', input_shape=self.input_shape)
else:
print("Not exists {}".format(self.model_name))
return None
if self.extractor:
for layer in pre_trained.layers:
layer.trainable = False
if self.model_name in ["VGG16", "vgg16", "VGG19", "vgg19"]:
x = Flatten()(pre_trained.output)
x = Dense(1024, activation="relu",
kernel_initializer="he_normal")(x)
x = Dropout(0.5)(x)
x = Dense(1024, activation="relu",
kernel_initializer="he_normal")(x)
x = Dropout(0.5)(x)
else:
x = GlobalAveragePooling2D()(pre_trained.output)
x = Flatten()(x)
y = Dense(2, activation="softmax")(x)
model = Model(inputs=pre_trained.input, outputs=y)
model.layers[0]._name = "input"
return model
def get_siamese_model(name=None, input_shape=(224, 224, 3),
embedding_vec_size=512, not_freeze_last=2):
"""
Model architecture
"""
if name == "InceptionV3":
base_model = inception_v3.InceptionV3(
weights='imagenet', include_top=False)
model_preprocess_input = inception_v3.preprocess_input
if name == "InceptionResNetV2":
base_model = inception_resnet_v2.InceptionResNetV2(
weights='imagenet', include_top=False)
model_preprocess_input = inception_resnet_v2.preprocess_input
if name == "DenseNet121":
base_model = densenet.DenseNet121(
weights='imagenet', include_top=False)
model_preprocess_input = densenet.preprocess_input
if name == "DenseNet169":
base_model = densenet.DenseNet169(
weights='imagenet', include_top=False)
model_preprocess_input = densenet.preprocess_input
if name == "DenseNet201":
base_model = densenet.DenseNet201(
weights='imagenet', include_top=False)
model_preprocess_input = densenet.preprocess_input
if name == "MobileNetV2":
base_model = mobilenet_v2.MobileNetV2(
weights='imagenet', include_top=False)
model_preprocess_input = mobilenet_v2.preprocess_input
if name == "MobileNet":
base_model = mobilenet.MobileNet(
weights='imagenet', include_top=False)
model_preprocess_input = mobilenet.preprocess_input
if name == "ResNet50":
base_model = resnet50.ResNet50(
weights='imagenet', include_top=False)
model_preprocess_input = resnet50.preprocess_input
if name == "VGG16":
base_model = vgg16.VGG16(
weights='imagenet', include_top=False)
model_preprocess_input = vgg16.preprocess_input
if name == "VGG19":
base_model = vgg19.VGG19(
weights='imagenet', include_top=False)
model_preprocess_input = vgg19.preprocess_input
if name == "Xception":
base_model = xception.Xception(
weights='imagenet', include_top=False)
model_preprocess_input = xception.preprocess_input
# Verifica se existe base_model
if 'base_model' not in locals():
return ["InceptionV3", "InceptionResNetV2",
"DenseNet121", "DenseNet169", "DenseNet201",
"MobileNetV2", "MobileNet",
"ResNet50",
"VGG16", "VGG19",
"Xception"
]
# desativando treinamento
for layer in base_model.layers[:-not_freeze_last]:
layer.trainable = False
x = base_model.layers[-1].output
x = GlobalAveragePooling2D()(x)
x = Dense(
embedding_vec_size,
activation='linear', # sigmoid? relu?
name='embedding',
use_bias=False
)(x)
model = Model(
inputs=base_model.input,
outputs=x
)
left_input = Input(input_shape)
right_input = Input(input_shape)
# Generate the encodings (feature vectors) for the two images
encoded_l = model(left_input)
encoded_r = model(right_input)
# Add a customized layer to compute the absolute difference between the encodings
L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
L1_distance = L1_layer([encoded_l, encoded_r])
# Add a dense layer with a sigmoid unit to generate the similarity score
prediction = Dense(
1,
activation=Activation(gaussian),
use_bias=False,
kernel_constraint=NonNeg()
)(L1_distance)
# Connect the inputs with the outputs
siamese_net = Model(
inputs=[left_input, right_input],
outputs=prediction
)
return {
"model": siamese_net,
"preprocess_input": model_preprocess_input
}
def __init__(
self,
cls: keras_hierarchicalclassification.KerasHierarchicalClassifier):
self.cls = cls
with configuration.ConfigurationContext("KerasIncrementalModel"):
# Preprocessing
self.random_crop_to_size = configuration.get(
"random_crop_to_size", None)
_channel_mean = configuration.get("channel_mean",
[127.5, 127.5, 127.5])
self.channel_mean_normalized = np.array(_channel_mean) / 255.0
_channel_stddev = configuration.get("channel_stddev",
[127.5, 127.5, 127.5])
self.channel_stddev_normalized = np.array(_channel_stddev) / 255.0
# Batch size
self.batchsize_max = configuration.get("batchsize_max", 256)
self.batchsize_min = configuration.get("batchsize_min", 1)
self.sequential_training_batches = configuration.get(
"sequential_training_batches", 1)
self.autobs_vram = configuration.get(
"autobs_vram", configuration.get_system("gpu0_vram"))
# Fine-tuning options
self.do_train_feature_extractor = configuration.get(
"train_feature_extractor", False)
self.use_pretrained_weights = configuration.get(
"use_pretrained_weights", "ILSVRC2012")
# Architecture
self.architecture = configuration.get("architecture",
"keras::ResNet50V2")
# Optimization and regularization
self.l2_regularization = configuration.get("l2_regularization",
5e-5)
self.optimizer_name = configuration.get("optimizer", "adam")
if self.optimizer_name == "sgd":
self.sgd_momentum = configuration.get("sgd_momentum", 0.9)
self.lr_schedule_cfg = configuration.get("lr_schedule", {
"name": "constant",
"config": {
"initial_lr": 0.003
}
})
self.lr_schedule = keras_learningrateschedule.get(
self.lr_schedule_cfg)
if self.architecture == "keras::ResNet50V2":
self.feature_extractor = resnet_v2.ResNet50V2(
include_top=False,
input_tensor=None,
input_shape=None,
pooling="avg",
weights="imagenet"
if self.use_pretrained_weights == "ILSVRC2012" else None,
)
self.pixels_per_gb = 1100000
self._add_regularizers()
elif self.architecture == "keras::InceptionResNetV2":
self.feature_extractor = inception_resnet_v2.InceptionResNetV2(
include_top=False,
input_tensor=None,
input_shape=None,
pooling="avg",
weights="imagenet"
if self.use_pretrained_weights == "ILSVRC2012" else None,
)
self.pixels_per_gb = 700000
self._add_regularizers()
elif self.architecture == "keras::MobileNetV2":
with configuration.ConfigurationContext("KerasIncrementalModel"):
self.side_length = configuration.get("side_length",
no_default=True)
self.feature_extractor = mobilenet_v2.MobileNetV2(
include_top=False,
input_tensor=None,
input_shape=(self.side_length, self.side_length, 3),
pooling="avg",
weights="imagenet"
if self.use_pretrained_weights == "ILSVRC2012" else None,
)
self.pixels_per_gb = 2000000
self._add_regularizers()
elif self.architecture == "keras::NASNetMobile":
with configuration.ConfigurationContext("KerasIncrementalModel"):
self.side_length = configuration.get("side_length",
no_default=True)
self.feature_extractor = nasnet.NASNetMobile(
include_top=False,
input_tensor=None,
input_shape=(self.side_length, self.side_length, 3),
pooling="avg",
weights="imagenet"
if self.use_pretrained_weights == "ILSVRC2012" else None,
)
self.pixels_per_gb = 1350000
self._add_regularizers()
elif self.architecture == "keras::CIFAR-ResNet56":
assert (self.do_train_feature_extractor
), "There are no pretrained weights for this architecture!"
assert (self.use_pretrained_weights is None
), "There are no pretrained weights for this architecture!"
from chia.methods.common import keras_cifar_resnet
self.feature_extractor = keras_cifar_resnet.feature_extractor(
version=2, n=6, l2_norm=self.l2_regularization)
self.pixels_per_gb = 200000
else:
raise ValueError(f'Unknown architecture "{self.architecture}"')
if self.optimizer_name == "adam":
self.optimizer = tf.keras.optimizers.Adam(self.lr_schedule(0))
else:
self.optimizer = tf.keras.optimizers.SGD(
learning_rate=self.lr_schedule(0), momentum=self.sgd_momentum)
self.augmentation = keras_dataaugmentation.KerasDataAugmentation()
if (self.use_pretrained_weights is not None
and self.use_pretrained_weights != "ILSVRC2012"):
print(
f"Loading alternative pretrained weights {self.use_pretrained_weights}"
)
self.feature_extractor.load_weights(self.use_pretrained_weights)
if not self.do_train_feature_extractor:
for layer in self.feature_extractor.layers:
layer.trainable = False
self.reported_auto_bs = False
# State here
self.current_step = 0
validation_generator = train_datagen.flow_from_directory(
images_dir,
target_size=(224, 224),
color_mode='rgb',
batch_size=32,
shuffle=True,
classes=imagenet_labels,
class_mode='sparse',
subset='validation')
###################
# Define transfer learning model
###################
base_model = mobilenet_v2.MobileNetV2(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
base_model.trainable = False
keras_model = keras.Sequential([
base_model,
keras.layers.GlobalAveragePooling2D(),
keras.layers.Dense(1024, activation='relu'),
keras.layers.Dense(512, activation='relu'),
keras.layers.Dropout(0.2),
keras.layers.Dense(5, activation='softmax')
])
keras_model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metric='accuracy')
def set_model(self, model_name, top_n=5):
if model_name == 'densenet':
self.model = densenet.DenseNet121(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
self.target_size = (224, 224)
self.decoder = lambda x: densenet.decode_predictions(x, top=top_n)
self.ref = """
<ul>
<li><a href='https://arxiv.org/abs/1608.06993' target='_blank'>
Densely Connected Convolutional Networks</a> (CVPR 2017 Best Paper Award)</li>
</ul>
"""
elif model_name == 'inception_resnet_v2':
self.model = inception_resnet_v2.InceptionResNetV2(
include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
self.target_size = (299, 299)
self.decoder = lambda x: inception_resnet_v2.decode_predictions(
x, top=top_n)
self.ref = """
<ul>
<li><a href='https://arxiv.org/abs/1602.07261' target='_blank'>
Inception-v4, Inception-ResNet and the Impact of Residual Connections on Learning</a></li>
</ul>
"""
elif model_name == 'inception_v3':
self.model = inception_v3.InceptionV3(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
self.target_size = (299, 299)
self.decoder = lambda x: inception_v3.decode_predictions(x,
top=top_n)
self.ref = """<ul>
<li><a href='https://arxiv.org/abs/1512.00567' target='_blank'>
Rethinking the Inception Architecture for Computer Vision</a></li>
</ul>
"""
elif model_name == 'mobilenet':
self.model = mobilenet.MobileNet(input_shape=None,
alpha=1.0,
depth_multiplier=1,
dropout=1e-3,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000)
self.target_size = (224, 224)
self.decoder = lambda x: mobilenet.decode_predictions(x, top=top_n)
self.ref = """<ul>
<li><a href='https://arxiv.org/abs/1704.04861' target='_blank'>
MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications</a></li>
</ul>
"""
elif model_name == 'mobilenet_v2':
self.model = mobilenet_v2.MobileNetV2(input_shape=None,
alpha=1.0,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000)
self.target_size = (224, 224)
self.decoder = lambda x: mobilenet_v2.decode_predictions(x,
top=top_n)
self.ref = """<ul>
<li><a href='https://arxiv.org/abs/1801.04381' target='_blank'>
MobileNetV2: Inverted Residuals and Linear Bottlenecks</a></li>
</ul>
"""
elif model_name == 'nasnet':
self.model = nasnet.NASNetLarge(input_shape=None,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000)
self.target_size = (224, 224)
self.decoder = lambda x: nasnet.decode_predictions(x, top=top_n)
self.ref = """<ul>
<li><a href='https://arxiv.org/abs/1707.07012' target='_blank'>
Learning Transferable Architectures for Scalable Image Recognition</a></li>
</ul>
"""
elif model_name == 'resnet50':
self.model = resnet50.ResNet50(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
self.target_size = (224, 224)
self.decoder = lambda x: resnet50.decode_predictions(x, top=top_n)
self.ref = """<ul>
<li>ResNet :
<a href='https://arxiv.org/abs/1512.03385' target='_blank'>Deep Residual Learning for Image Recognition
</a></li>
</ul>
"""
elif model_name == 'vgg16':
self.model = vgg16.VGG16(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
self.target_size = (224, 224)
self.decoder = lambda x: vgg16.decode_predictions(x, top=top_n)
self.ref = """<ul>
<li><a href='https://arxiv.org/abs/1409.1556' target='_blank'>
Very Deep Convolutional Networks for Large-Scale Image Recognition</a></li>
</ul>"""
elif model_name == 'vgg19':
self.model = vgg19.VGG19(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
self.target_size = (224, 224)
self.decoder = lambda x: vgg19.decode_predictions(x, top=top_n)
self.ref = """<ul>
<li><a href='https://arxiv.org/abs/1409.1556' target='_blank'>Very Deep Convolutional Networks for Large-Scale Image Recognition</a></li>
</ul>"""
elif model_name == 'xception':
self.model = xception.Xception(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
self.target_size = (299, 299)
self.decoder = lambda x: xception.decode_predictions(x, top=top_n)
self.ref = """<ul>
<li><a href='https://arxiv.org/abs/1610.02357' target='_blank'>Xception: Deep Learning with Depthwise Separable Convolutions</a></li>
</ul>"""
else:
logger.ERROR('There has no model name !!!')
def load_my_model():
model = tfApps.MobileNetV2(weights='imagenet')
model.summary()
return model
result = create_mask(pred)
cv2.imwrite('result.jpg',result)
# cv2.imshow('input image',sample_image[0])
# cv2.imshow('input image',sample_image)
# cv2.imshow('result',result)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
def normalize(input_image, input_mask):
input_image = tf.cast(input_image, tf.float32) / 255.0
input_mask -= 1
return input_image, input_mask
OUTPUT_CHANNELS = 3
base_model = mobilenet_v2.MobileNetV2(input_shape=(128,128,3),include_top=False)
# Use the activations of these layers
layer_names = [
'block_1_expand_relu', # 64x64
'block_3_expand_relu', # 32x32
'block_6_expand_relu', # 16x16
'block_13_expand_relu', # 8x8
'block_16_project', # 4x4
]
layers = [base_model.get_layer(name).output for name in layer_names]
# Create the feature extraction model
down_stack = tf.keras.Model(inputs=base_model.input, outputs=layers)
down_stack.trainable = False
def mnv2_hm_r_v2(self, inp_shape):
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model.layers.pop()
inp = mobilenet_model.input
'''heatmap can not be generated from activation layers, so we use out_relu'''
x = mobilenet_model.get_layer('out_relu').output # 7, 7, 1280
x = Deconvolution2D(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer='he_uniform')(x) # 14, 14, 256
x = BatchNormalization()(x)
x = ReLU()(x)
x = Deconvolution2D(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer='he_uniform')(x) # 28, 28, 256
x = BatchNormalization()(x)
x = ReLU()(x)
x = Deconvolution2D(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
kernel_initializer='he_uniform')(x) # 56, 56, 256
bn_0 = BatchNormalization(name='bn_0')(x)
x = ReLU()(bn_0)
'''reduce to 7'''
x = Conv2D(256,
kernel_size=(3, 3),
strides=(2, 2),
padding='same',
kernel_initializer='he_uniform')(x) # 28, 28, 256
bn_1 = BatchNormalization(name='bn_1')(x) # 28, 28, 256
x = ReLU()(bn_1)
x = Conv2D(256,
kernel_size=(3, 3),
strides=(2, 2),
padding='same',
kernel_initializer='he_uniform')(x) # 14, 14, 256
bn_2 = BatchNormalization(name='bn_2')(x) # 14, 14, 256
x = ReLU()(bn_2)
x = Conv2D(256,
kernel_size=(3, 3),
strides=(2, 2),
padding='same',
kernel_initializer='he_uniform')(x) # 7, 7 , 256
bn_3 = BatchNormalization(name='bn_3')(x) # 7, 7 , 256
x = ReLU()(bn_3)
'''increase to 56'''
x = Deconvolution2D(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
name='deconv1',
kernel_initializer='he_uniform')(x) # 14, 14, 256
x = BatchNormalization()(x)
x = keras.layers.add([x, bn_2]) # 14, 14, 256
x = ReLU()(x)
x = Deconvolution2D(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
name='deconv2',
kernel_initializer='he_uniform')(x) # 28, 28, 256
x = BatchNormalization()(x)
x = keras.layers.add([x, bn_1]) # 28, 28, 256
x = ReLU()(x)
x = Deconvolution2D(filters=256,
kernel_size=(4, 4),
strides=(2, 2),
padding='same',
name='deconv3',
kernel_initializer='he_uniform')(x) # 56, 56, 256
x = BatchNormalization()(x)
x = keras.layers.add([x, bn_0]) # 56, 56, 256
'''out_conv_layer'''
out_heatmap = Conv2D(LearningConfig.landmark_len // 2,
kernel_size=1,
padding='same',
name='out_heatmap')(x)
revised_model = Model(inp, [
out_heatmap,
])
revised_model.summary()
# plot_model(revised_model, to_file='mnv2_hm.png', show_shapes=True, show_layer_names=True)
model_json = revised_model.to_json()
with open("mnv2_hm_r_v2.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def mn_asm_v1(self, tensor):
# block_13_project_BN block_10_project_BN block_6_project_BN
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=None,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=tensor,
pooling=None)
mobilenet_model.layers.pop()
inp = mobilenet_model.input
# '''block_1 { block_6_project_BN 14, 14, 46 '''
# x = mobilenet_model.get_layer('block_6_project_BN').output # 14, 14, 46
# x = Deconvolution2D(filters=128, kernel_size=(2, 2), strides=(2, 2), padding='same', activation='relu',
# name='block_1_deconv_1', kernel_initializer='he_uniform')(x) # 28, 28, 128
# x = BatchNormalization(name='block_1_out_bn_1')(x)
#
# x = Deconvolution2D(filters=128, kernel_size=(2, 2), strides=(2, 2), padding='same', activation='relu',
# name='block_1_deconv_2', kernel_initializer='he_uniform')(x) # 56, 56, 128
# x = BatchNormalization(name='block_1_out_bn_2')(x)
#
# block_1_out = Conv2D(LearningConfig.landmark_len // 2, kernel_size=1, padding='same', name='block_1_out')(x)
# '''block_1 }'''
'''block_2 { block_10_project_BN 14, 14, 96 '''
x = mobilenet_model.get_layer(
'block_10_project_BN').output # 14, 14, 96
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='block_2_deconv_1',
kernel_initializer='he_uniform')(x) # 28, 28, 128
x = BatchNormalization(name='block_2_out_bn_1')(x)
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='block_2_deconv_2',
kernel_initializer='he_uniform')(x) # 56, 56, 128
x = BatchNormalization(name='block_2_out_bn_2')(x)
block_2_out = Conv2D(LearningConfig.landmark_len // 2,
kernel_size=1,
padding='same',
name='block_2_out')(x)
'''block_2 }'''
'''block_3 { block_13_project_BN 7, 7, 160 '''
x = mobilenet_model.get_layer(
'block_13_project_BN').output # 7, 7, 160
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='block_3_deconv_1',
kernel_initializer='he_uniform')(x) # 14, 14, 128
x = BatchNormalization(name='block_3_out_bn_1')(x)
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='block_3_deconv_2',
kernel_initializer='he_uniform')(x) # 28, 28, 128
x = BatchNormalization(name='block_3_out_bn_2')(x)
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='block_3_deconv_3',
kernel_initializer='he_uniform')(x) # 56, 56, 128
x = BatchNormalization(name='block_3_out_bn_3')(x)
block_3_out = Conv2D(LearningConfig.landmark_len // 2,
kernel_size=1,
padding='same',
name='block_3_out')(x)
'''block_3 }'''
'''heatmap can not be generated from activation layers, so we use out_relu'''
x = mobilenet_model.get_layer('out_relu').output # 7, 7, 1280
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv1',
kernel_initializer='he_uniform')(x) # 14, 14, 256
x = BatchNormalization(name='out_bn1')(x)
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv2',
kernel_initializer='he_uniform')(x) # 28, 28, 256
x = BatchNormalization(name='out_bn2')(x)
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv3',
kernel_initializer='he_uniform')(x) # 56, 56, 256
x = BatchNormalization(name='out_bn3')(x)
out_heatmap = Conv2D(LearningConfig.landmark_len // 2,
kernel_size=1,
padding='same',
name='out_heatmap')(x)
revised_model = Model(
inp,
[
# block_1_out, # 85
# block_2_out, # 90
# block_3_out, # 97
out_heatmap # 100
])
revised_model.summary()
# plot_model(revised_model, to_file='mnv2_hm.png', show_shapes=True, show_layer_names=True)
model_json = revised_model.to_json()
with open("mn_asm_v1.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def mn_asm_v0(self, tensor):
"""
has only one output
we use custom loss for this network and using ASM to correct points after that
"""
# block_13_project_BN block_10_project_BN block_6_project_BN
mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=None,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=tensor,
pooling=None)
mobilenet_model.layers.pop()
inp = mobilenet_model.input
'''heatmap can not be generated from activation layers, so we use out_relu'''
x = mobilenet_model.get_layer('out_relu').output # 7, 7, 1280
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv1',
kernel_initializer='he_uniform')(x) # 14, 14, 256
x = BatchNormalization(name='out_bn1')(x)
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv2',
kernel_initializer='he_uniform')(x) # 28, 28, 256
x = BatchNormalization(name='out_bn2')(x)
x = Deconvolution2D(filters=128,
kernel_size=(2, 2),
strides=(2, 2),
padding='same',
activation='relu',
name='deconv3',
kernel_initializer='he_uniform')(x) # 56, 56, 256
x = BatchNormalization(name='out_bn3')(x)
out_heatmap = Conv2D(LearningConfig.point_len,
kernel_size=1,
padding='same',
name='out_heatmap')(x)
revised_model = Model(inp, out_heatmap)
revised_model.summary()
# plot_model(revised_model, to_file='mnv2_hm.png', show_shapes=True, show_layer_names=True)
model_json = revised_model.to_json()
with open("mn_asm_v0.json", "w") as json_file:
json_file.write(model_json)
return revised_model
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
import cv2
from tensorflow.keras.applications import mobilenet_v2
# Set up figure
fig, ax = plt.subplots()
vid = plt.imshow(np.ones((224,224,3)))
lbl = plt.text(5, 20, "Loading...", size=20)
lbl.set_bbox({'facecolor':'white', 'alpha':0.5})
ax.set_axis_off()
plt.ion()
plt.show()
# Set up neural network
model = mobilenet_v2.MobileNetV2(weights='imagenet')
# Set up video capture
cap = cv2.VideoCapture(0)
fig.canvas.mpl_connect('close_event', lambda evt : cap.release())
# Function to call for each frame
def update(i):
# Capture frame from webcam
ret, frame_bgr = cap.read()
assert ret
frame = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
min_dim = np.min(frame.shape[0:2])
frame = cv2.resize(frame[0:min_dim, 0:min_dim, :], (224, 224))
vid.set_data(frame)
def create_model(input_shape=(384, 576, 3)) -> Model:
i = layers.Input(shape=input_shape)
x = tf.cast(i, tf.float32)
x = mobilenet_v2.preprocess_input(x)
backbone = mobilenet_v2.MobileNetV2(input_tensor=x,
include_top=False,
weights='imagenet')
backbone.trainable = False
x = backbone(x)
# Get layers and attach FPN
backbone_scale_2 = backbone.get_layer('expanded_conv_project_BN').output
backbone_scale_3 = backbone.get_layer('block_2_add').output
backbone_scale_4 = backbone.get_layer('block_5_add').output
backbone_scale_5 = backbone.get_layer('block_12_add').output
backbone_scale_6 = x
# backbone_scale_6 = backbone.get_layer('block_16_project_BN').output
# FPN
fpn_d = 128
# m7 = layers.Conv2D(fpn_d, 1, padding='same')(backbone_scale_7)
# p7 = m7
# p7_up = layers.UpSampling2D(size=(32,32))(p7)
m6 = layers.Conv2D(fpn_d, 1, padding='same', name='m6')(backbone_scale_6)
p6 = layers.Conv2D(fpn_d, 3, padding='same', name='p6')(m6)
p6 = layers.Conv2D(fpn_d, 3, padding='same')(p6)
# p6 = layers.Conv2D(fpn_d, 3, padding='same')(p6)
p6_up = layers.UpSampling2D(size=(16, 16))(p6)
y = layers.Conv2D(fpn_d, 1, padding='same')(backbone_scale_5)
z = layers.UpSampling2D()(m6)
m5 = layers.Add(name='m5', )([y, z])
p5 = layers.Conv2D(fpn_d, 3, padding='same', name='p5')(m5)
p5 = layers.Conv2D(fpn_d, 3, padding='same')(p5)
# p5 = layers.Conv2D(fpn_d, 3, padding='same')(p5)
p5_up = layers.UpSampling2D(size=(8, 8))(p5)
y = layers.Conv2D(fpn_d, 1, padding='same')(backbone_scale_4)
z = layers.UpSampling2D()(m5)
m4 = layers.Add(name='m4', )([y, z])
p4 = layers.Conv2D(fpn_d, 3, padding='same', name='p4')(m4)
p4 = layers.Conv2D(fpn_d, 3, padding='same')(p4)
# p4 = layers.Conv2D(fpn_d, 3, padding='same')(p4)
p4_up = layers.UpSampling2D(size=(4, 4))(p4)
y = layers.Conv2D(fpn_d, 1, padding='same')(backbone_scale_3)
z = layers.UpSampling2D()(m4)
m3 = layers.Add(name='m3', )([y, z])
p3 = layers.Conv2D(fpn_d, 3, padding='same', name='p3')(m3)
p3 = layers.Conv2D(fpn_d, 3, padding='same')(p3)
# p3 = layers.Conv2D(fpn_d, 3, padding='same')(p3)
p3_up = layers.UpSampling2D()(p3)
y = layers.Conv2D(fpn_d, 1, padding='same')(backbone_scale_2)
z = layers.UpSampling2D()(m3)
m2 = layers.Add(name='m2', )([y, z])
p2 = layers.Conv2D(fpn_d, 3, padding='same', name='p2')(m2)
p2 = layers.Conv2D(fpn_d, 3, padding='same')(p2)
# p2 = layers.Conv2D(fpn_d, 3, padding='same')(p2)
# Concatenate Outputs of FPN
# x = layers.Concatenate()([p3, p4_up, p5_up, p6_up])
x = layers.Concatenate()([p2, p3_up, p4_up, p5_up, p6_up])
x = layers.Dropout(0.1)(x)
# Boil it down to four heatmaps
x = layers.Conv2D(fpn_d, 3, padding='same')(x)
x = layers.Dropout(0.1)(x)
x = layers.Conv2D(fpn_d, 3, padding='same')(x)
x = layers.Dropout(0.1)(x)
x = layers.Conv2D(fpn_d, 3, padding='same')(x)
# x = layers.Conv2D(fpn_d, 3, padding='same', activation='relu')(x)
x = layers.LeakyReLU(alpha=0.1)(x)
x = layers.Conv2D(1, 1, padding='same')(x)
# x = layers.Conv2D(5, 1, padding='same')(x)
# x = layers.UpSampling2D(interpolation='bilinear', size=(4,4))(x)
model = Model(inputs=i, outputs=x)
return model, backbone
