def create_model(self):
"""
input_tensor = Input(shape=self.input_shape)
base_model = NASNetMobile(include_top=False, input_tensor=input_tensor)
x = base_model(input_tensor)
out_1 = GlobalMaxPooling2D()(x)
out_2 = GlobalAveragePooling2D()(x)
out_3 = Flatten()(x)
out = Concatenate(axis=-1)([out_1, out_2, out_3])
out = Dropout(0.5)(out)
out = Dense(57, activation="sigmoid")(out)
model = Model(input_tensor, out)
model.compile(optimizer=Adam(self.learning_rate), loss=binary_crossentropy, metrics=['acc'])
"""
# Efficientnet
input_tensor = Input(shape=self.input_shape)
base_model = efn.EfficientNetB3(weights='imagenet',
include_top=True,
input_tensor=input_tensor)
x = base_model(input_tensor)
x = Flatten()(x)
x = Dense(1024, activation="relu")(x)
x = Dropout(0.5)(x)
out = Dense(62, activation="sigmoid")(x)
model = Model(input_tensor, out)
model.compile(optimizer=SGD(self.learning_rate),
loss='binary_crossentropy',
metrics=['acc'])
return model
def get_model():
with strategy.scope():
margin = ArcMarginProduct(
n_classes=N_CLASSES,
s=30,
m=0.5,
name='head/arc_margin',
dtype='float32'
)
inp = tf.keras.layers.Input(shape=(*IMAGE_SIZE, 3), name='inp1')
label = tf.keras.layers.Input(shape=(), name='inp2')
x = efn.EfficientNetB3(weights='imagenet', include_top=False)(inp)
x = tf.keras.layers.GlobalAveragePooling2D()(x)
x = margin([x, label])
output = tf.keras.layers.Softmax(dtype='float32')(x)
model = tf.keras.models.Model(inputs=[inp, label], outputs=[output])
opt = tf.keras.optimizers.Adam(learning_rate=LR)
model.compile(
optimizer=opt,
loss=[tf.keras.losses.SparseCategoricalCrossentropy()],
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]
)
return model
def build_efficientnet_b3(input_shape, n_classes=4, print_model_summary=True):
efficientnet_model = efn.EfficientNetB3(
weights="imagenet", # 'noisy-student',
input_shape=input_shape,
classes=n_classes,
pooling="avg",
include_top=False,
)
# Define the input layer.
cnn_input = Input(shape=input_shape, name="cnn_input")
# Create a feature vector using the pre-trained EfficientNet model.
x = efficientnet_model(cnn_input)
# Add MLP block to generate a prediction.
x = Dense(1024, activation="relu")(x)
output = Dense(n_classes,
activation="sigmoid" if n_classes == 1 else "softmax")(x)
model = Model(inputs=[cnn_input], outputs=[output])
if print_model_summary:
print(model.summary())
return model
def build_backbone_net_graph(input_tensor, architecture, weights=None):
"""
Build basic feature extraction networks.
:param input_tensor: Input of the basic networks, should be a tensor or tf.keras.layers.Input
:param architecture: The architecture name of the basic network.
:param weights: Whether download and initialize weights from the pre-trained weights,
could be either 'imagenet', (pre-training on ImageNet)
'noisy-student',
'None' (random initialization)，
or the path to the weights file to be loaded。
:return: Efficient Model and corresponding endpoints.
"""
assert architecture in ['efficientnet-b0', 'efficientnet-b1',
'efficientnet-b2', 'efficientnet-b3',
'efficientnet-b4', 'efficientnet-b5',
'efficientnet-b7', 'efficientnet-b7',
'efficientnet-l2']
if architecture == 'efficientnet-b0':
return efn.EfficientNetB0(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
elif architecture == 'efficientnet-b1':
return efn.EfficientNetB1(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
elif architecture == 'efficientnet-b2':
return efn.EfficientNetB2(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
elif architecture == 'efficientnet-b3':
return efn.EfficientNetB3(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
elif architecture == 'efficientnet-b4':
return efn.EfficientNetB4(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
elif architecture == 'efficientnet-b5':
return efn.EfficientNetB5(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
elif architecture == 'efficientnet-b6':
return efn.EfficientNetB6(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
elif architecture == 'efficientnet-b7':
return efn.EfficientNetB7(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
elif architecture == 'efficientnet-l2':
return efn.EfficientNetL2(include_top=False, weights=weights,
input_tensor=input_tensor,
input_shape=[None, None, 3])
else:
raise ValueError("Argument architecture should in "
"[efficientnet-b0, efficientnet-b1, "
"efficientnet-b2, efficientnet-b3, efficientnet-b4, efficientnet-b5, "
"efficientnet-b7, efficientnet-b7, efficientnet-l2] "
"but get %s" % architecture)
def effnet_retinanet(num_classes, backbone='EfficientNetB0', inputs=None, modifier=None, **kwargs):
""" Constructs a retinanet model using a resnet backbone.
Args
num_classes: Number of classes to predict.
backbone: Which backbone to use (one of ('resnet50', 'resnet101', 'resnet152')).
inputs: The inputs to the network (defaults to a Tensor of shape (None, None, 3)).
modifier: A function handler which can modify the backbone before using it in retinanet (this can be used to freeze backbone layers for example).
Returns
RetinaNet model with a ResNet backbone.
"""
# choose default input
if inputs is None:
if keras.backend.image_data_format() == 'channels_first':
inputs = keras.layers.Input(shape=(3, None, None))
else:
# inputs = keras.layers.Input(shape=(224, 224, 3))
inputs = keras.layers.Input(shape=(None, None, 3))
# get last conv layer from the end of each block [28x28, 14x14, 7x7]
if backbone == 'EfficientNetB0':
model = efn.EfficientNetB0(input_tensor=inputs, include_top=False, weights=None)
elif backbone == 'EfficientNetB1':
model = efn.EfficientNetB1(input_tensor=inputs, include_top=False, weights=None)
elif backbone == 'EfficientNetB2':
model = efn.EfficientNetB2(input_tensor=inputs, include_top=False, weights=None)
elif backbone == 'EfficientNetB3':
model = efn.EfficientNetB3(input_tensor=inputs, include_top=False, weights=None)
elif backbone == 'EfficientNetB4':
model = efn.EfficientNetB4(input_tensor=inputs, include_top=False, weights=None)
elif backbone == 'EfficientNetB5':
model = efn.EfficientNetB5(input_tensor=inputs, include_top=False, weights=None)
elif backbone == 'EfficientNetB6':
model = efn.EfficientNetB6(input_tensor=inputs, include_top=False, weights=None)
elif backbone == 'EfficientNetB7':
model = efn.EfficientNetB7(input_tensor=inputs, include_top=False, weights=None)
else:
raise ValueError('Backbone (\'{}\') is invalid.'.format(backbone))
layer_outputs = ['block4a_expand_activation', 'block6a_expand_activation', 'top_activation']
layer_outputs = [
model.get_layer(name=layer_outputs[0]).output, # 28x28
model.get_layer(name=layer_outputs[1]).output, # 14x14
model.get_layer(name=layer_outputs[2]).output, # 7x7
]
# create the densenet backbone
model = keras.Model(inputs=inputs, outputs=layer_outputs, name=model.name)
# invoke modifier if given
if modifier:
model = modifier(model)
# create the full model
return retinanet.retinanet(inputs=inputs, num_classes=num_classes, backbone_layers=model.outputs, **kwargs)
def efficientnetb3(self, input_shape, **kwagrs):
# x_input = tf.keras.Input(shape=input_shape, name="triplet")
m = efn.EfficientNetB3(input_shape=(256, 256, 3), weights=None)
# remove the output layer, leave the feature extraction part
m_fe = tf.keras.Model(inputs=m.inputs, outputs=m.layers[-2].output)
output = tf.keras.layers.Dense(1, activation="sigmoid")(m_fe.output)
m = tf.keras.Model(inputs=m_fe.inputs, outputs=output)
return m
def get_efficientnet_model(model_version):
if model_version == "B0": return efn.EfficientNetB0(weights='imagenet')
elif model_version == "B1": return efn.EfficientNetB1(weights='imagenet')
elif model_version == "B2": return efn.EfficientNetB2(weights='imagenet')
elif model_version == "B3": return efn.EfficientNetB3(weights='imagenet')
elif model_version == "B4": return efn.EfficientNetB4(weights='imagenet')
elif model_version == "B5": return efn.EfficientNetB5(weights='imagenet')
elif model_version == "B6": return efn.EfficientNetB6(weights='imagenet')
elif model_version == "B7": return efn.EfficientNetB7(weights='imagenet')
else: return efn.EfficientNetB0(weights='imagenet')
def get_train_tfrecord(num_shards, train_captions, img_name_vector,
file_index):
print("Total number of TFrecords: ", num_shards, flush=True)
# Using InceptionV3 and using the pretrained Imagenet weights
image_model = efn.EfficientNetB3(
weights=
"noisy-student", # Choose between imagenet and 'noisy-student'
# weights='imagenet',
input_shape=target_size,
include_top=False,
)
new_input = image_model.input
hidden_layer = image_model.layers[-1].output
image_features_extract_model = tf.keras.Model(new_input, hidden_layer)
for i in range(num_shards):
subsets_num = int(len(train_captions) / num_shards)
sub_split_img_id = img_name_vector[i * subsets_num:(i + 1) *
subsets_num]
sub_split_cap_train = train_captions[i * subsets_num:(i + 1) *
subsets_num]
tfrecord_name = "1Mio_TFR/" + "train-%02d.tfrecord" % file_index
writer = tf.io.TFRecordWriter(tfrecord_name)
counter = 0
for j in range(len(sub_split_img_id)):
img = tf.expand_dims(load_image(sub_split_img_id[j])[0], 0)
batch_features = image_features_extract_model(img)
batch_features = tf.reshape(
batch_features,
(batch_features.shape[0], -1, batch_features.shape[3]),
)
# print(batch_features.shape)
# print(decoded_image.shape)
caption_ = sub_split_cap_train[j]
# image_id_ = sub_split_img_id[counter]
counter = counter + 1
feature = {
#'image_id': _bytes_feature(image_id_.encode('utf8')),
"image_raw":
_bytes_feature(batch_features.numpy().tostring()),
"caption": _bytes_feature(caption_.tostring()),
}
example = tf.train.Example(features=tf.train.Features(
feature=feature))
serialized = example.SerializeToString()
writer.write(serialized)
print("%s write to tfrecord success!" % tfrecord_name, flush=True)
file_index = file_index + 1
def _create_efficientNet(self):
initializer = tf.keras.initializers.glorot_uniform()
eff_net = efn.EfficientNetB3(
include_top=True,
weights=None,
input_tensor=None,
input_shape=[
InputDataSize.image_input_size, InputDataSize.image_input_size,
3
],
pooling=None,
classes=LearningConfig.expression_output_len_shrunk
) # or weights='noisy-student'
'''save json'''
# model_json = eff_net.to_json()
#
# with open("./model_archs/eff_net_b0.json", "w") as json_file:
# json_file.write(model_json)
'''revise model'''
eff_net.layers.pop()
x = eff_net.get_layer('top_dropout').output
# o_velocity = Dense(LearningConfig.velocity_output_len, name='O_Velocity')(x)
o_expression = Dense(LearningConfig.expression_output_len_shrunk,
activation='linear',
name='O_Expression_sh')(x)
# o_velocity = Dense(1, name='O_Velocity_reg')(x)
# x = GlobalAveragePooling2D()(x)
# x = Dropout(0.2)(x)
# x = Dense(512, use_bias=False, kernel_initializer=initializer)(x)
# x = BatchNormalization()(x)
# x = ReLU()(x)
# x = Dropout(0.2)(x)
# x = Dense(256, use_bias=False, kernel_initializer=initializer)(x)
# x = BatchNormalization()(x)
# x = ReLU()(x)
# '''create outputs'''
# o_velocity = Dense(1, name='O_Velocity_reg')(x)
# o_velocity = Dense(LearningConfig.velocity_output_len, name='O_Velocity_categorical')(x)
# o_category = Dense(output_len, name='O_Category')(x)
inp = eff_net.input
revised_model = Model(inp, [o_expression])
# revised_model = Model(inp, [o_velocity, o_expression])
# revised_model = Model(inp, [o_velocity, o_category])
revised_model.summary()
'''save json'''
model_json = revised_model.to_json()
with open("./model_archs/ef_b3.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def load_image_features_extract_model(target_size):
# Using EfficientnetB3 and using the pretrained Imagenet weights
image_model = efn.EfficientNetB3(weights='noisy-student',
input_shape=target_size,
include_top=False)
new_input = image_model.input
hidden_layer = image_model.layers[-1].output
image_features_extract_model = tf.keras.Model(new_input, hidden_layer)
return image_features_extract_model
def get_efficientnet(self):
models_dict ={
'b0': efn.EfficientNetB0(input_shape=self.shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=self.shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=self.shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=self.shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=self.shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=self.shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=self.shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=self.shape,weights=None,include_top=False)
}
return models_dict[self.model_class]
def __init__(self):
# Create the base model from the pre-trained EfficientNet
model_full = efn.EfficientNetB3(input_shape=(self.IMG_SIZE,
self.IMG_SIZE, 3),
include_top=False,
weights="imagenet")
model_full.trainable = False
self.model = tf.keras.Model(
model_full.inputs,
model_full.get_layer(self.LAYER_NAME).output)
self.model.trainable = False
def get_model(model_bucket, model_filename):
logger.debug('loading model')
try:
if model_bucket is None:
model = efn.EfficientNetB3(input_shape=(256, 256, 3),
weights=None,
classes=101)
logger.debug('loaded dummy model')
else:
logger.debug('downloading model file')
client = storage.Client.create_anonymous_client()
bucket = client.get_bucket(model_bucket)
blob = bucket.get_blob(model_filename)
if '.h5' in model_filename:
try:
path = '/tmp/model.h5'
blob.download_to_filename(path)
except FileNotFoundError:
path = 'model.h5'
blob.download_to_filename(path)
logger.debug('loading model')
model = tf.keras.models.load_model(path, compile=False)
elif '.tflite' in model_filename:
try:
path = '/tmp/model.tflite'
blob.download_to_filename(path)
except FileNotFoundError:
path = 'model.tflite'
blob.download_to_filename(path)
logger.debug('loading tflite model')
interpreter = tf.lite.Interpreter(model_path=path)
interpreter.allocate_tensors()
model = interpreter
except:
raise
return model
def __init__(self, hparams):
super(InputEmbedding, self).__init__()
self.hparams = hparams
if hparams.base_model_name == 'InceptionV3':
base_model = tf.keras.applications.InceptionV3(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'InceptionResNetV2':
base_model = tf.keras.applications.InceptionResNetV2(
include_top=False, weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'EfficientNetB0':
base_model = efn.EfficientNetB0(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'EfficientNetB1':
base_model = efn.EfficientNetB1(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'EfficientNetB2':
base_model = efn.EfficientNetB2(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'EfficientNetB3':
base_model = efn.EfficientNetB3(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'EfficientNetB4':
base_model = efn.EfficientNetB4(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'EfficientNetB5':
base_model = efn.EfficientNetB5(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'EfficientNetB6':
base_model = efn.EfficientNetB6(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
elif hparams.base_model_name == 'EfficientNetB7':
base_model = efn.EfficientNetB7(include_top=False,
weights='imagenet')
base_model_layers = [layer.name for layer in base_model.layers]
assert hparams.end_point in base_model_layers, "no {} layer in {}".format(
hparams.end_point, hparams.base_model_name)
conv_tower_output = base_model.get_layer(hparams.end_point).output
self.conv_model = tf.keras.models.Model(inputs=base_model.input,
outputs=conv_tower_output)
self.conv_out_shape = self.conv_model.predict(
np.array([np.zeros(hparams.image_shape)])).shape
self.encode_cordinate = EncodeCordinate(
input_shape=self.conv_out_shape)
def create_b3(include_top=False,
input_shape=None,
input_tensor=None,
weights="noisy-student"):
"""ネットワークの作成。"""
import efficientnet.tfkeras as efn
return efn.EfficientNetB3(
include_top=include_top,
input_shape=input_shape,
input_tensor=input_tensor,
weights=weights,
)
def __init__(self, embedding_dim):
super(CNNEncoder, self).__init__()
self.cnn0 = efn.EfficientNetB3(weights='noisy-student',
input_shape=TARGET_SIZE,
include_top=False)
# e.g. layers[-1].output = TensorShape([None, 10, 10, 1536]) for
# B3 (not global pooling)
self.cnn = Model(self.cnn0.input, self.cnn0.layers[-1].output)
self.cnn.trainable = False
# shape after fc == (batch_size, attention_features_shape,
# embedding_dim) >> this is my mistake, should be hidden instead
# of embedding_dim
self.fc = Dense(embedding_dim)
def create_efficientNet(self,
inp_shape,
input_tensor,
output_len,
is_teacher=True):
if is_teacher: # for teacher we use a heavier network
eff_net = efn.EfficientNetB3(include_top=True,
weights=None,
input_tensor=None,
input_shape=[
InputDataSize.image_input_size,
InputDataSize.image_input_size, 3
],
pooling=None,
classes=output_len)
# return self._create_efficientNet_3deconv(inp_shape, input_tensor, output_len)
else: # for student we use the small network
eff_net = efn.EfficientNetB0(
include_top=True,
weights=None,
input_tensor=None,
input_shape=inp_shape,
pooling=None,
classes=output_len) # or weights='noisy-student'
eff_net.layers.pop()
inp = eff_net.input
x = eff_net.get_layer('top_activation').output
x = GlobalAveragePooling2D()(x)
x = keras.layers.Dropout(rate=0.5)(x)
output = Dense(output_len, activation='linear', name='out')(x)
eff_net = Model(inp, output)
eff_net.summary()
# plot_model(eff_net, to_file='eff_net.png', show_shapes=True, show_layer_names=True)
# tf.keras.utils.plot_model(
# eff_net,
# to_file="eff_net.png",
# show_shapes=False,
# show_layer_names=True,
# rankdir="TB"
# )
# model_json = eff_net.to_json()
# with open("eff_net.json", "w") as json_file:
# json_file.write(model_json)
return eff_net
def getEffTFModel(self, n=0):
modelInput = tf.keras.Input(batch_input_shape=(None, 5, self.config['net_size'], self.config['net_size'], 3))
modelInput0, modelInput1, modelInput2, modelInput3, modelInput4 = tf.split(modelInput, [1, 1, 1, 1, 1], 1)
x0 = tf.squeeze(tf.keras.layers.Lambda(lambda x0: x0)(modelInput0))
x1 = tf.squeeze(tf.keras.layers.Lambda(lambda x1: x1)(modelInput1))
x2 = tf.squeeze(tf.keras.layers.Lambda(lambda x2: x2)(modelInput2))
x3 = tf.squeeze(tf.keras.layers.Lambda(lambda x3: x3)(modelInput3))
x4 = tf.squeeze(tf.keras.layers.Lambda(lambda x4: x4)(modelInput4))
net = ''
if n % 10 == 0:
net = efn.EfficientNetB0(include_top=False, weights='imagenet', input_shape=(self.config['net_size'], self.config['net_size'], 3), pooling='avg')
elif n % 10 == 1:
net = efn.EfficientNetB1(include_top=False, weights='imagenet', input_shape=(self.config['net_size'], self.config['net_size'], 3), pooling='avg')
elif n % 10 == 2:
net = efn.EfficientNetB2(include_top=False, weights='imagenet', input_shape=(self.config['net_size'], self.config['net_size'], 3), pooling='avg')
elif n % 10 == 3:
net = efn.EfficientNetB3(include_top=False, weights='imagenet', input_shape=(self.config['net_size'], self.config['net_size'], 3), pooling='avg')
elif n % 10 == 4:
net = efn.EfficientNetB4(include_top=False, weights='imagenet', input_shape=(self.config['net_size'], self.config['net_size'], 3), pooling='avg')
activation = tf.keras.layers.LeakyReLU()
self.config['rnn_size'] = 256
ret0 = net(x0)
ret0 = Dense(self.config['rnn_size'], activation=activation)(ret0)
ret0 = tf.expand_dims(ret0, axis=1)
ret0 = self.transformer(ret0)
ret1 = net(x1)
ret1 = Dense(self.config['rnn_size'], activation=activation)(ret1)
ret1 = tf.expand_dims(ret1, axis=1)
ret1 = self.transformer(ret1)
ret2 = net(x2)
ret2 = Dense(self.config['rnn_size'], activation=activation)(ret2)
ret2 = tf.expand_dims(ret2, axis=1)
ret2 = self.transformer(ret2)
ret3 = net(x3)
ret3 = Dense(self.config['rnn_size'], activation=activation)(ret3)
ret3 = tf.expand_dims(ret3, axis=1)
ret3 = self.transformer(ret3)
ret4 = net(x4)
ret4 = Dense(self.config['rnn_size'], activation=activation)(ret4)
ret4 = tf.expand_dims(ret4, axis=1)
ret4 = self.transformer(ret4)
ret = tf.concat([ret0, ret1, ret2, ret3, ret4], axis=1)
print(ret)
x = tf.keras.layers.Dense(self.config['rnn_size'], activation=activation)(ret)
model = tf.keras.Model(modelInput, x)
return model
def build_base_efficientnet_model(input_shape,
n_classes=4,
print_model_summary=True):
model = Sequential([
efn.EfficientNetB3(
input_shape=(3, 512, 512),
weights="imagenet",
include_top=False,
),
GlobalAveragePooling2D(),
Dense(n_classes, activation="softmax", dtype="float32"),
])
if print_model_summary:
print(model.summary())
return model
def model_chooser(self, classes=2, weights=None):
print("Model selection started.")
name = self.model_name
if(name=='C0'):
self.model = efn.EfficientNetB0(include_top=True, weights=weights, classes=classes)
elif(name=='C1'):
self.model = efn.EfficientNetB1(include_top=True, weights=weights, classes=classes)
elif(name=='C2'):
self.model = efn.EfficientNetB2(include_top=True, weights=weights, classes=classes)
elif(name=='C3'):
self.model = efn.EfficientNetB3(include_top=True, weights=weights, classes=classes)
elif(name=='C4'):
self.model = efn.EfficientNetB4(include_top=True, weights=weights, classes=classes)
elif(name=='C5'):
self.model = efn.EfficientNetB5(include_top=True, weights=weights, classes=classes)
if(classes==2):
self.model.compile(optimizer="adam", loss="binary_crossentropy", metrics = ['acc'])
elif(classes>2):
self.model.compile(optimizer="adam", loss="categorical_crossentropy", metrics = ['acc'])
def create_base_model(base_model_name, pretrained=True, IMAGE_SIZE=[300, 300]):
if pretrained is False:
weights = None
else:
weights = "imagenet"
if base_model_name == 'B0':
base = efn.EfficientNetB0(weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, 3])
elif base_model_name == 'B1':
base = efn.EfficientNetB1(weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, 3])
elif base_model_name == 'B2':
base = efn.EfficientNetB2(weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, 3])
elif base_model_name == 'B3':
base = efn.EfficientNetB3(weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, 3])
elif base_model_name == 'B4':
base = efn.EfficientNetB4(weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, 3])
elif base_model_name == 'B5':
base = efn.EfficientNetB5(weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, 3])
elif base_model_name == 'B6':
base = efn.EfficientNetB6(weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, 3])
elif base_model_name == 'B7':
base = efn.EfficientNetB7(weights=weights,
include_top=False,
input_shape=[*IMAGE_SIZE, 3])
base = remove_dropout(base)
base.trainable = True
return base
def get_effnet(shape, weights, num_classes):
input = layers.Input(shape=shape)
base_model = efn.EfficientNetB3(weights=weights,
include_top=False,
input_shape=shape)
base_model.trainable = True
output = base_model(input)
output = layers.GlobalMaxPooling2D()(output)
output = layers.Dense(256)(output)
output = layers.LeakyReLU(alpha=0.25)(output)
output = layers.Dropout(0.25)(output)
output = layers.Dense(16, activation="relu")(output)
output = layers.Dropout(0.15)(output)
output = layers.Dense(num_classes, activation="sigmoid")(output)
model = Model(input, output)
return model
def create_efficientNet(self,
inp_shape,
input_tensor,
output_len,
is_teacher=True):
if is_teacher: # for teacher we use a heavier network
eff_net = efn.EfficientNetB3(include_top=True,
weights=None,
input_tensor=None,
input_shape=[
InputDataSize.image_input_size,
InputDataSize.image_input_size, 3
],
pooling=None,
classes=output_len)
else: # for student we use the small network
eff_net = efn.EfficientNetB0(
include_top=True,
weights=None,
input_tensor=None,
input_shape=inp_shape,
pooling=None,
classes=output_len) # or weights='noisy-student'
eff_net.layers.pop()
inp = eff_net.input
x = eff_net.get_layer('top_activation').output
x = GlobalAveragePooling2D()(x)
x = keras.layers.Dropout(rate=0.5)(x)
output = Dense(output_len, activation='linear', name='out')(x)
eff_net = Model(inp, output)
eff_net.summary()
return eff_net
def load_b3():
cnn_net = efn.EfficientNetB3(weights='imagenet',include_top=False,input_shape=(300, 300, 3))
model = build_model(cnn_net)
model.load_weights('models/effnet_b3.h5')
return model
def create_model_efficientnet(self, mode=None):
"""
Create one of the efficientnet models (by now its EfficientNetB5)
Arguments:
- mode - mode of architecture to be trained
f.e linear means that we use linear head,
non linear means that we use more dense layers with relu activation
and meta means that we build model that includes both metadata and image data.
Returns:
- Model - Keras model that is ready to be fit.
"""
if mode == 'meta':
input_tensor = Input(shape=self.input_shape[0], name='inp1')
input_meta = Input(shape=self.input_shape[1], name='inp2')
else:
input_tensor = Input(shape=self.input_shape)
base_model = efn.EfficientNetB3(include_top=False,
input_tensor=input_tensor)
x = base_model(input_tensor)
if mode == 'linear':
x = GlobalAveragePooling2D()(x)
x = Dense(2048, activation='linear')(x)
x = Dense(self.output_shape, activation='sigmoid')(x)
model = Model(input_tensor, x)
model.compile(optimizer=self.optimizer(self.learning_rate),
loss=self.loss,
metrics=self.metric)
model.summary()
return model
elif mode == 'non_linear':
out1 = AdaptiveMaxPooling2D((2, 2))(x)
out2 = AdaptiveAveragePooling2D((2, 2))(x)
out = Concatenate(axis=-1)([out1, out2])
out = Flatten()(out)
out = Dense(2048, activation='relu')(out)
out = Dropout(0.4)(out)
out = Dense(1024, activation='relu')(out)
out = Dropout(0.3)(out)
out = Dense(512, activation='relu')(out)
out = Dropout(0.2)(out)
out = Dense(256, activation='relu')(out)
out = Dropout(0.1)(out)
out = Dense(self.output_shape, activation='sigmoid')(out)
model = Model(input_tensor, out)
model.compile(optimizer=self.optimizer(self.learning_rate),
loss=self.loss,
metrics=self.metric)
model.summary()
return model
elif mode == 'meta':
x = GlobalAveragePooling2D()(x)
x = Dense(2048, activation='linear')(x)
x = BatchNormalization()(x)
x1 = Dense(128)(input_meta)
x1 = LeakyReLU()(x1)
x1 = BatchNormalization()(x1)
concat = Concatenate()([x, x1])
x = Dense(1000, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(300, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(80, activation='relu')(x)
x1 = Dense(60, activation='linear')(x)
x1 = BatchNormalization()(x1)
x1 = Dense(20, activation='linear')(x1)
x1 = BatchNormalization()(x1)
concat = Dense(1300, activation='relu')(concat)
concat = BatchNormalization()(concat)
concat = Dense(400, activation='relu')(concat)
concat = BatchNormalization()(concat)
concat = Dense(70, activation='relu')(concat)
concat_layer = Concatenate()([x, x1, concat])
last_head = Dense(20, activation='relu')(concat_layer)
last_head = BatchNormalization()(last_head)
output = Dense(1, activation='sigmoid')(last_head)
model = Model([input_tensor, input_meta], output)
model.compile(optimizer=self.optimizer(self.learning_rate),
loss=self.loss,
metrics=self.metric)
model.summary()
return model
def build(name, width, height, depth, n_classes, reg=0.8):
"""
Args:
name: name of the network
width: width of the images
height: height of the images
depth: number of channels of the images
reg: regularization value
"""
# If Keras backend is TensorFlow
inputShape = (height, width, depth)
chanDim = -1
# If Keras backend is Theano
if K.image_data_format() == "channels_first":
inputShape = (depth, height, width)
chanDim = 1
# Define the base model architecture
if name == 'EfficientNetB0':
base_model = efn.EfficientNetB0(weights='imagenet',
include_top=False,
input_shape=inputShape)
elif name == 'EfficientNetB1':
base_model = efn.EfficientNetB1(weights='imagenet',
include_top=False,
input_shape=inputShape)
elif name == 'EfficientNetB2':
base_model = efn.EfficientNetB2(weights='imagenet',
include_top=False,
input_shape=inputShape)
elif name == 'EfficientNetB3':
base_model = efn.EfficientNetB3(weights='imagenet',
include_top=False,
input_shape=inputShape)
elif name == 'EfficientNetB4':
base_model = efn.EfficientNetB4(weights='imagenet',
include_top=False,
input_shape=inputShape)
elif name == 'EfficientNetB5':
base_model = efn.EfficientNetB5(weights='imagenet',
include_top=False,
input_shape=inputShape)
elif name == 'EfficientNetB6':
base_model = efn.EfficientNetB6(weights='imagenet',
include_top=False,
input_shape=inputShape)
elif name == 'ResNet50':
base_model = ResNet50(weights='imagenet',
include_top=False,
input_shape=inputShape)
elif name == 'DenseNet121':
base_model = DenseNet121(weights='imagenet',
include_top=False,
input_shape=inputShape)
#x1 = GlobalMaxPooling2D()(base_model.output) # Compute the max pooling of the base model output
#x2 = GlobalAveragePooling2D()(base_model.output) # Compute the average pooling of the base model output
#x3 = Flatten()(base_model.output) # Flatten the base model output
#x = Concatenate(axis=-1)([x1, x2, x3])
x = GlobalAveragePooling2D()(base_model.output)
x = Dropout(0.5)(x)
"""
# First Dense => Relu => BN => DO
fc_layer_1 = Dense(512, kernel_regularizer=l2(reg))(x)
activation_1 = Activation('relu')(fc_layer_1)
batch_norm_1 = BatchNormalization(axis=-1)(activation_1)
dropout_1 = Dropout(0.5)(batch_norm_1)
# First Dense => Relu => BN => DO
fc_layer_2 = Dense(256, kernel_regularizer=l2(reg))(dropout_1)
activation_2 = Activation('relu')(fc_layer_2)
batch_norm_2 = BatchNormalization(axis=-1)(activation_2)
dropout_2 = Dropout(0.5)(batch_norm_2)
# Add the output layer
output = Dense(n_classes, kernel_regularizer=l2(reg), activation='softmax')(dropout_2)
"""
output = Dense(n_classes,
kernel_regularizer=l2(reg),
activation='softmax')(x)
# Create the model
model = Model(inputs=base_model.inputs, outputs=output)
return model
def get_model(arch="b3", pretrained="imagenet", image_size=(128, 128, 3)):
image_input = tf.keras.layers.Input(shape=image_size,
dtype='float32',
name='image_input')
if arch.startswith("b2"):
base_model = efn.EfficientNetB2(weights=pretrained,
input_shape=image_size,
include_top=False)
elif arch.startswith("b3"):
base_model = efn.EfficientNetB3(weights=pretrained,
input_shape=image_size,
include_top=False)
elif arch.startswith("b4"):
base_model = efn.EfficientNetB4(weights=pretrained,
input_shape=image_size,
include_top=False)
elif arch.startswith("b5"):
base_model = efn.EfficientNetB5(weights=pretrained,
input_shape=image_size,
include_top=False)
elif arch.startswith("b6"):
base_model = efn.EfficientNetB6(weights=pretrained,
input_shape=image_size,
include_top=False)
elif arch.startswith("b7"):
base_model = efn.EfficientNetB7(weights=pretrained,
input_shape=image_size,
include_top=False)
else:
raise ValueError("Unknown arch!")
base_model.trainable = True
tmp = base_model(image_input)
hidden_dim = base_model.output_shape[-1]
tmp = tf.keras.layers.GlobalAveragePooling2D()(tmp)
tmp = tf.keras.layers.Dropout(0.5)(tmp)
if arch.endswith("g"):
prediction_0 = tf.keras.layers.Dense(CLASS_COUNTS[0],
activation='softmax',
name="root",
dtype='float32')(SELayer(
hidden_dim, 8)(tmp))
prediction_1 = tf.keras.layers.Dense(CLASS_COUNTS[1],
activation='softmax',
name="vowel",
dtype='float32')(SELayer(
hidden_dim, 8)(tmp))
prediction_2 = tf.keras.layers.Dense(CLASS_COUNTS[2],
activation='softmax',
name="consonant",
dtype='float32')(SELayer(
hidden_dim, 8)(tmp))
else:
prediction_0 = tf.keras.layers.Dense(CLASS_COUNTS[0],
activation='softmax',
name="root",
dtype='float32')(tmp)
prediction_1 = tf.keras.layers.Dense(CLASS_COUNTS[1],
activation='softmax',
name="vowel",
dtype='float32')(tmp)
prediction_2 = tf.keras.layers.Dense(CLASS_COUNTS[2],
activation='softmax',
name="consonant",
dtype='float32')(tmp)
prediction = tf.keras.layers.Concatenate(axis=-1)(
[prediction_0, prediction_1, prediction_2])
return tf.keras.Model(image_input, prediction)
def TL(n_classes,
num_frozen_layers,
optimizer_type="sgd",
lr=.001,
reg=1e-6,
dropout_chance=0.2,
use_focal_loss=False,
channels=1,
output="label_bbox",
model_type="resnet"):
if model_type == "resnet":
base = ResNet50(weights="imagenet",
include_top=False,
input_shape=(224, 224, channels))
elif model_type == "mobilenet":
base = MobileNetV2(weights="imagenet",
include_top=False,
input_shape=(224, 224, channels))
elif model_type == "efn_b3":
base = efn.EfficientNetB3(weights="imagenet",
include_top=False,
input_shape=(224, 224, channels))
x = GlobalAvgPool2D()(base.output)
# =============================================================================
# drop = Dropout(dropout_chance)(x)
# dense = Dense(units=512, kernel_regularizer=l2(reg))(drop)
# norm = BatchNormalization()(dense)
# x = Activation(activation="relu")(norm)
# =============================================================================
drop = Dropout(dropout_chance)(x)
class_output = Dense(n_classes,
kernel_regularizer=l2(reg),
activation="softmax",
name="labels")(drop)
bbox_output = Dense(units=4, name="bbox")(drop)
for i, layer in enumerate(base.layers):
if i < num_frozen_layers:
layer.trainable = False
#print(i, layer)
# Optimizer
if optimizer_type == "sgd":
optimizer = SGD(lr=lr, momentum=0.9, decay=0.01)
elif optimizer_type == "nesterov_sgd":
optimizer = SGD(lr=lr, momentum=0.9, decay=0.01, nesterov=True)
elif optimizer_type == "adam":
optimizer = Adam(lr=lr)
model = None
if output == "bbox":
model = Model(inputs=base.input, outputs=bbox_output)
model.compile(loss="msle", optimizer=optimizer, metrics=["accuracy"])
elif output == "label":
loss = focal_loss(alpha=1) if use_focal_loss \
else "categorical_crossentropy"
model = Model(inputs=base.input, outputs=class_output)
model.compile(loss=loss, optimizer=optimizer, metrics=["accuracy"])
else:
loss = focal_loss(alpha=1) if use_focal_loss \
else "categorical_crossentropy"
model = Model(inputs=base.input, outputs=[class_output, bbox_output])
model.compile(loss=[loss, "msle"],
loss_weights=[.8, .2],
optimizer=optimizer,
metrics=["accuracy"])
print(model.summary())
return model
application.config['SQLALCHEMY_DATABASE_URI'] = \
'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{db}'.format(
user=db_username,
passwd=db_password ,
host=db_hostname,
port=db_port,
db=db_database)
application.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
application.config['SECRET_KEY'] = os.urandom(24)
db = SQLAlchemy(application)
bootstrap = Bootstrap(application)
login_manager = LoginManager()
login_manager.init_app(application)
base_model = efn.EfficientNetB3(weights='imagenet')
model = Model(inputs=base_model.input,
outputs=base_model.get_layer('top_dropout').output)
target_size = (300, 300)
@login_manager.user_loader
def load_user(id):
"""Reload the user object from the user ID stored in the session"""
return User.query.get(int(id))
@application.route('/', methods=['GET', 'POST'])
def index():
"""upload a file from a client machine."""
plt.show()
# ## Train Model
# ## Use pretrain model
# - source : [efficientnet-b3_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5](https://github.com/Callidior/keras-applications/releases/download/efficientnet/efficientnet-b3_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5)
# In[24]:
#pretrain_model = 'model_history/efficientnet/efficientnet-b3_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
pretrain_model = 'model_history/efficientnet/efficientnet-b3_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
bottleneck = efn.EfficientNetB3(
input_shape=(int(SEQ_LEN ** .5) * IMG_SIZE, int(SEQ_LEN ** .5) * IMG_SIZE, 3),
weights=pretrain_model,
include_top=False,
pooling='avg'
)
bottleneck = Model(inputs=bottleneck.inputs, outputs=bottleneck.layers[-2].output)
model = Sequential()
model.add(bottleneck)
model.add(GlobalAveragePooling2D())
model.add(Flatten())
model.add(BatchNormalization())
model.add(Dropout(.2))
model.add(Dense(512, activation='elu'))
model.add(BatchNormalization())
model.add(Dropout(.2))
model.add(Dense(6, activation='sigmoid'))
model.summary()
