def Recognition_model():
input = Input(shape=(64, 64, 3))
x = Conv2D(filters=(64), kernel_size=(64, 21), strides=1)(input)
x = BatchNormalization()(x)
x1 = GlobalMaxPool2D()(x)
x = Conv2D(filters=(64), kernel_size=(21, 64), strides=1)(input)
x = BatchNormalization()(x)
x2 = GlobalMaxPool2D()(x)
x = Conv2D(filters=(64), kernel_size=(21, 21), strides=1)(input)
x = BatchNormalization()(x)
x3 = GlobalMaxPool2D()(x)
x = Conv2D(filters=(64), kernel_size=(5, 5), strides=1)(input)
x = BatchNormalization()(x)
x4 = GlobalMaxPool2D()(x)
# x5 = Flatten()(x1+x2+x3+x4)
x = Dense(1512)(x5)
output = Dense(2350, activation="softmax")(x)
model = Model(inputs=input, outputs=output)
model.summary()
model.compile(optimizer='adam',
loss="categorical_crossentropy",
metrics=['acc'])
return model
def baseline_model():
input_1 = Input(shape=(None, None, 3))
input_2 = Input(shape=(None, None, 3))
base_model = MobileNetV2(weights="imagenet", include_top=False)
x1 = base_model(input_1)
x2 = base_model(input_2)
x1 = Concatenate(axis=-1)([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)])
x2 = Concatenate(axis=-1)([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)])
x3 = Subtract()([x1, x2])
x3 = Multiply()([x3, x3])
x = Multiply()([x1, x2])
x = Concatenate(axis=-1)([x, x3])
x = Dense(100, activation="relu")(x)
x = Dropout(0.01)(x)
out = Dense(1, activation="sigmoid")(x)
model = Model([input_1, input_2], out)
model.compile(loss="binary_crossentropy", metrics=[acc], optimizer=Adam(0.00001))
model.summary()
return model
def MolMapDualPathNet(molmap1_size,
molmap2_size,
n_outputs=1,
conv1_kernel_size=13,
dense_layers=[256, 128, 32],
dense_avf='relu',
last_avf=None):
"""
parameters
----------------------
molmap1_size: w, h, c, shape of first molmap
molmap2_size: w, h, c, shape of second molmap
n_outputs: output units
dense_layers: list, how many dense layers and units
dense_avf: activation function for dense layers
last_avf: activation function for last layer
"""
tf.keras.backend.clear_session()
## first inputs
d_inputs1 = Input(molmap1_size)
d_conv1 = Conv2D(48,
conv1_kernel_size,
padding='same',
activation='relu',
strides=1)(d_inputs1)
d_pool1 = MaxPool2D(pool_size=3, strides=2, padding='same')(d_conv1) #p1
d_incept1 = Inception(d_pool1, strides=1, units=32)
d_pool2 = MaxPool2D(pool_size=3, strides=2, padding='same')(d_incept1) #p2
d_incept2 = Inception(d_pool2, strides=1, units=64)
d_flat1 = GlobalMaxPool2D()(d_incept2)
## second inputs
f_inputs1 = Input(molmap2_size)
f_conv1 = Conv2D(48,
conv1_kernel_size,
padding='same',
activation='relu',
strides=1)(f_inputs1)
f_pool1 = MaxPool2D(pool_size=3, strides=2, padding='same')(f_conv1) #p1
f_incept1 = Inception(f_pool1, strides=1, units=32)
f_pool2 = MaxPool2D(pool_size=3, strides=2, padding='same')(f_incept1) #p2
f_incept2 = Inception(f_pool2, strides=1, units=64)
f_flat1 = GlobalMaxPool2D()(f_incept2)
## concat
x = Concatenate()([d_flat1, f_flat1])
## dense layer
for units in dense_layers:
x = Dense(units, activation=dense_avf)(x)
## last layer
outputs = Dense(n_outputs, activation=last_avf)(x)
model = tf.keras.Model(inputs=[d_inputs1, f_inputs1], outputs=outputs)
return model
def __init__(self, base, preprocess):
'''
Wrapper class for a CNN model that augments incoming data, uses a pre-trained model as a
a base and is trained to perform binary classification.
Class arguments:
base -- one of the available models from tensorflow.keras.applications
preprocess -- the corresponding preprocess function from the model's module
'''
# Instantiate base model
base_model = base(include_top=False, input_shape=(DIM, DIM, 3))
# Build model
inputs = Input(shape=(DIM, DIM, 3))
x = augment(inputs)
x = preprocess(x)
x = base_model(x)
x = GlobalMaxPool2D()(x)
x = Dropout(0.4)(x)
outputs = Dense(1, activation='sigmoid')(x)
# Compile model
self.model = Model(inputs, outputs, name=base.__name__)
self.model.compile(optimizer=Adam(lr=LEARNING_RATE),
loss="binary_crossentropy",
metrics=['accuracy'])
def get_recognizer():
i = 6
inputs = Input((None, None, 3))
conv1 = Conv2D(2**i, 3, padding="same", activation="selu")(inputs)
conv1 = Conv2D(2**i, 3, padding="same", activation="selu")(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(2 * 2**i, 3, padding="same", activation="selu")(pool1)
conv2 = Conv2D(2 * 2**i, 3, padding="same", activation="selu")(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(4 * 2**i, 3, padding="same", activation="selu")(pool2)
conv3 = Conv2D(4 * 2**i, 3, padding="same", activation="selu")(conv3)
pool3 = GlobalMaxPool2D()(conv3)
d_out = Dense(10, activation="softmax")(pool3)
model = Model(inputs=[inputs], outputs=[d_out])
model.compile(
optimizer=Adam(lr=1e-5),
loss=losses.sparse_categorical_crossentropy,
metrics=["acc"],
)
return model
def malley_cnn_40(input_shape, n_classes):
X_input = Input(input_shape)
X = Lambda(lambda q: expand_dims(q, -1), name='expand_dims')(X_input)
X = Conv2D(64, [3, 7], padding='same')(X)
X = Activation('relu')(X)
X = MaxPool2D([2, 1])(X)
X = Conv2D(128, [7, 1], padding='same')(X)
X = Activation('relu')(X)
X = MaxPool2D([4, 1])(X)
X = Conv2D(256, [5, 1], padding='valid')(X)
X = Activation('relu')(X)
X = Conv2D(512, [1, 7], padding='same')(X)
X = Activation('relu')(X)
X = GlobalMaxPool2D()(X)
X = Dense(512, activation='relu')(X)
X = Dropout(0.5)(X)
X = Dense(n_classes, activation='softmax')(X)
model = Model(inputs=X_input, outputs=X)
return model
def build_COVIDNet(num_classes=3, flatten=True, checkpoint='',args=None):
if args.model == 'resnet50v2':
base_model = ResNet50V2(include_top=False, weights='imagenet', input_shape=(args.img_size, args.img_size, 3))
x = base_model.output
if args.model =='mobilenetv2':
base_model = MobileNetV2(include_top=False, weights='imagenet', input_shape=(args.img_size, args.img_size, 3))
x = base_model.output
if args.model == 'custom':
base_model = covidnet(input_tensor=None, input_shape=(args.img_size, args.img_size, 3), classes=3)
x = base_model.output
if args.model == 'EfficientNet':
import efficientnet.tfkeras as efn
base_model = efn.EfficientNetB4(weights=None, include_top=True, input_shape=(args.img_size, args.img_size, 3), classes=3)
x = base_model.output
if flatten:
x = Flatten()(x)
else:
# x = GlobalAveragePooling2D()(x)
x = GlobalMaxPool2D()(x)
if args.datapipeline == 'covidx':
x = Dense(1024, activation='relu',kernel_regularizer=tf.keras.regularizers.l2(0.0001))(x)
x = Dense(256, activation='relu',kernel_regularizer=tf.keras.regularizers.l2(0.0001))(x)
# x = Dropout(0.2)(x)
predictions = Dense(num_classes, activation='softmax',name=f'FC_{num_classes}')(x)
model = Model(inputs=base_model.input, outputs=predictions)
if len(checkpoint):
model.load_weights(checkpoint)
return model
def channel_attention(inputs):
"""Channel Attention Map calculation
The function aims to implement a simple
version of a Channel Attention, whose output
is a tensor that will weight each channel of the
input.
Args:
inputs: Input tensor, above which the channel
map will be calculated
Returns:
Channel Attention map
"""
max_features = GlobalMaxPool2D()(inputs)
avg_features = GlobalAvgPool2D()(inputs)
extracted_max = extraction_network(max_features)
extracted_avg = extraction_network(avg_features)
merge = Add()[extracted_avg, extracted_max]
return reshape(sigmoid(merge), (-1, 1, 1, inputs.shape[3]))
def create_model(image_shape=(224, 224, 3),
restart_checkpoint=None,
backbone='mobilnetv2',
feature_len=128,
freeze=False):
"""
Creates an image encoder.
Args:
image_shape: input image shape (use [None, None] for resizable network)
restart_checkpoint: snapshot to be restored
backbone: the backbone CNN (one of mobilenetv2, densent121, custom)
feature_len: the length of the additional feature layer
freeze: freeze the backbone
"""
input_img = Input(shape=image_shape)
# add the backbone
backbone_name = backbone
if backbone_name == 'densenet121':
print('Using DenseNet121 backbone.')
backbone = DenseNet121(input_tensor=input_img, include_top=False)
backbone.layers.pop()
if freeze:
for layer in backbone.layers:
layer.trainable = False
backbone = backbone.output
elif backbone_name == 'mobilenetv2':
print('Using MobileNetV2 backbone.')
backbone = MobileNetV2(input_tensor=input_img, include_top=False)
backbone.layers.pop()
if freeze:
for layer in backbone.layers:
layer.trainable = False
backbone = backbone.output
elif backbone_name == 'custom':
backbone = custom_backbone(input_tensor=input_img)
else:
raise Exception('Unknown backbone: {}'.format(backbone_name))
# add the head layers
gmax = GlobalMaxPool2D()(backbone)
gavg = GlobalAvgPool2D()(backbone)
gmul = Multiply()([gmax, gavg])
ggavg = Average()([gmax, gavg])
backbone = Concatenate()([gmax, gavg, gmul, ggavg])
backbone = BatchNormalization()(backbone)
backbone = Dense(feature_len)(backbone)
backbone = Activation('sigmoid')(backbone)
encoder = Model(input_img, backbone)
if restart_checkpoint:
print('Loading weights from {}'.format(restart_checkpoint))
encoder.load_weights(restart_checkpoint,
by_name=True,
skip_mismatch=True)
return encoder
def initialize_model():
from model import Vggface2_ResNet50
# Set basic environments.
# Initialize GPUs
toolkits.initialize_GPU()
# ==> loading the pre-trained model.
input1 = Input(shape=(224, 224, 3))
input2 = Input(shape=(224, 224, 3))
# x1 = resnet.resnet50_backend(input1)
# x2 = resnet.resnet50_backend(input2)
base_model = Vggface2_ResNet50(include_top=False)
base_model.load_weights(weight_file, by_name=True)
print("successfully load model ", weight_file)
for x in base_model.layers:
x.trainable = True
x1 = base_model(input1)
x2 = base_model(input2)
x1 = Concatenate(axis=-1)([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)])
x2 = Concatenate(axis=-1)([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)])
x3 = Subtract()([x1, x2])
x3 = Multiply()([x3, x3])
x1_ = Multiply()([x1, x1])
x2_ = Multiply()([x2, x2])
x4 = Subtract()([x1_, x2_])
x = Concatenate(axis=-1)([x4, x3])
x = Dense(100, activation="relu")(x)
x = Dropout(0.3)(x)
x = Dense(25, activation="relu")(x)
x = Dropout(0.3)(x)
out = Dense(1, activation="sigmoid")(x)
model = Model([input1, input2], out)
# for x in model.layers[-21:]:
# x.trainable = True
model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam(0.00005))
model.summary()
return model
def build_model():
base_model = DenseNet201(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
x = base_model.output
x = GlobalMaxPool2D()(x)
x = Dropout(0.7)(x)
predictions = Dense(len(class_names), activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=predictions)
return model
def discriminator_model(input_shape, n_filters, kernel_size, multiplier, summary):
filters = n_filters
model = Sequential(name='Discriminator')
model.add(InputLayer(input_shape=input_shape))
for i in range(0, multiplier):
model.add(Conv2D(filters=filters, kernel_size=kernel_size, strides=2, padding='same'))
model.add(LeakyReLU(alpha=0.2))
filters = filters*2
model.add(GlobalMaxPool2D())
model.add(Dense(1, activation='sigmoid'))
if(summary):
model.summary()
return model
def get_model():
nclass = 10
inp = Input(shape=(None, None, 1))
img_1 = Convolution2D(64,
kernel_size=3,
activation=activations.relu,
padding="same")(inp)
img_1 = Convolution2D(64,
kernel_size=3,
activation=activations.relu,
padding="same")(img_1)
img_1 = MaxPool2D(pool_size=2)(img_1)
img_1 = Dropout(rate=0.1)(img_1)
img_1 = Convolution2D(128,
kernel_size=3,
activation=activations.relu,
padding="same")(img_1)
img_1 = Convolution2D(128,
kernel_size=3,
activation=activations.relu,
padding="same")(img_1)
img_1 = MaxPool2D(pool_size=2)(img_1)
img_1 = Dropout(rate=0.1)(img_1)
img_1 = Convolution2D(256,
kernel_size=3,
activation=activations.relu,
padding="valid")(img_1)
img_1 = Convolution2D(256,
kernel_size=3,
activation=activations.relu,
padding="valid")(img_1)
img_1 = GlobalMaxPool2D()(img_1)
img_1 = Dropout(rate=0.1)(img_1)
dense_1 = Dense(64, activation=activations.relu, name="dense_1")(img_1)
dense_1 = PermaDropout(rate=0.1)(dense_1)
dense_1 = Dense(64, activation=activations.relu, name="dense_2")(dense_1)
dense_1 = PermaDropout(rate=0.1)(dense_1)
dense_1 = Dense(nclass, activation=activations.softmax)(dense_1)
model = models.Model(inputs=inp, outputs=dense_1)
opt = optimizers.Adam(0.0001)
model.compile(optimizer=opt,
loss=losses.sparse_categorical_crossentropy,
metrics=['acc'])
#model.summary()
return model
def conv_block(inputs, filter_num, reduction_ratio, stride=1):
x = inputs
x = Conv2D(filter_num[0], (1, 1), strides=stride, padding='same')(x)
x = BatchNormalization(axis=3)(x)
x = Activation('relu')(x)
x = Conv2D(filter_num[1], (3, 3), strides=1, padding='same')(x)
x = BatchNormalization(axis=3)(x)
x = Activation('relu')(x)
x = Conv2D(filter_num[2], (1, 1), strides=1, padding='same')(x)
x = BatchNormalization(axis=3)(x)
# Channel Attention mudule
avgpool = GlobalAveragePooling2D()(x) # channel avgpool
maxpool = GlobalMaxPool2D()(x) # channel maxpool
# Shared MLP
Dense_layer1 = Dense(filter_num[2] // reduction_ratio,
activation='relu') # channel fc1
Dense_layer2 = Dense(filter_num[2], activation='relu') # channel fc2
avg_out = Dense_layer2(Dense_layer1(avgpool))
max_out = Dense_layer2(Dense_layer1(maxpool))
channel = tf.keras.layers.add([avg_out, max_out])
channel = Activation('sigmoid')(channel) # channel sigmoid
channel = Reshape((1, 1, filter_num[2]))(channel)
channel_out = tf.multiply(x, channel)
# Spatial Attention mudule
avgpool = tf.reduce_mean(channel_out, axis=3,
keepdims=True) # spatial avgpool
maxpool = tf.reduce_max(channel_out, axis=3,
keepdims=True) # spatial maxpool
spatial = Concatenate(axis=3)([avgpool, maxpool])
# kernel filter 7x7 follow the paper
spatial = Conv2D(1, (7, 7), strides=1,
padding='same')(spatial) # spatial conv2d
spatial_out = Activation('sigmoid')(spatial) # spatial sigmoid
CBAM_out = tf.multiply(channel_out, spatial_out)
# residual connection
r = Conv2D(filter_num[2], (1, 1), strides=stride, padding='same')(inputs)
x = tf.keras.layers.add([CBAM_out, r])
x = Activation('relu')(x)
return x
def __init__(self, num_classes, filter_size, hidden_dim):
super(CNN, self).__init__(name='cnn')
self.num_classes = num_classes
self.filter_size = filter_size
#Create Convolutional Filters
self.conv = Conv2D(filters=filter_size, kernel_size=(28,28), padding='same', activation='relu')
self.relu = Activation(activation='relu')
self.batch_norm = BatchNormalization()
self.pool = GlobalMaxPool2D(data_format='channels_last')
#Create FC layers
self.fc1 = Dense(units=hidden_dim, activation='relu')
self.Dropout = Dropout(rate=DROPOUT_P)
self.fc2 = Dense(units=num_classes, activation='softmax')
def MolMapNet(input_shape,
n_outputs=1,
conv1_kernel_size=13,
dense_layers=[128, 32],
dense_avf='relu',
last_avf=None):
"""
parameters
----------------------
molmap_shape: w, h, c
n_outputs: output units
dense_layers: list, how many dense layers and units
dense_avf: activation function for dense layers
last_avf: activation function for last layer
"""
assert len(input_shape) == 3
inputs = Input(input_shape)
conv1 = Conv2D(48,
conv1_kernel_size,
padding='same',
activation='relu',
strides=1)(inputs)
conv1 = MaxPool2D(pool_size=3, strides=2, padding='same')(conv1) #p1
incept1 = Inception(conv1, strides=1, units=32)
incept1 = MaxPool2D(pool_size=3, strides=2, padding='same')(incept1) #p2
incept2 = Inception(incept1, strides=1, units=64)
#flatten
x = GlobalMaxPool2D()(incept2)
## dense layer
for units in dense_layers:
x = Dense(units, activation=dense_avf)(x)
#last layer
outputs = Dense(n_outputs, activation=last_avf)(x)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
return model
def MolMapAddPathNet(molmap_shape,
additional_shape,
n_outputs=1,
dense_layers=[128, 32],
dense_avf='relu',
last_avf=None):
"""
parameters
----------------------
molmap_shape: w, h, c
n_outputs: output units
dense_layers: list, how many dense layers and units
dense_avf: activation function for dense layers
last_avf: activation function for last layer
"""
tf.keras.backend.clear_session()
assert len(molmap_shape) == 3
inputs = Input(molmap_shape)
inputs_actvie_amount = Input(additional_shape)
conv1 = Conv2D(48, 13, padding='same', activation='relu',
strides=1)(inputs)
conv1 = MaxPool2D(pool_size=3, strides=2, padding='same')(conv1) #p1
incept1 = Inception(conv1, strides=1, units=32)
incept1 = MaxPool2D(pool_size=3, strides=2, padding='same')(incept1) #p2
incept2 = Inception(incept1, strides=1, units=64)
#flatten
x = GlobalMaxPool2D()(incept2)
x = tf.keras.layers.concatenate([x, inputs_actvie_amount], axis=-1)
## dense layer
for units in dense_layers:
x = Dense(units, activation=dense_avf)(x)
#last layer
outputs = Dense(n_outputs, activation=last_avf)(x)
model = tf.keras.Model(inputs=[inputs, inputs_actvie_amount],
outputs=outputs)
return model
def model(vocab_size, lr=0.0001):
input_1 = Input(shape=(None, None, 3))
input_2 = Input(shape=(None, ))
input_3 = Input(shape=(None, ))
base_model = ResNet50(weights='imagenet', include_top=False)
x1 = base_model(input_1)
x1 = GlobalMaxPool2D()(x1)
dense_1 = Dense(vec_dim, activation="linear", name="dense_image_1")
x1 = dense_1(x1)
embed = Embedding(vocab_size, 50, name="embed")
gru = Bidirectional(GRU(256, return_sequences=True), name="gru_1")
dense_2 = Dense(vec_dim, activation="linear", name="dense_text_1")
x2 = embed(input_2)
x2 = SpatialDropout1D(0.1)(x2)
x2 = gru(x2)
x2 = GlobalMaxPool1D()(x2)
x2 = dense_2(x2)
x3 = embed(input_3)
x3 = SpatialDropout1D(0.1)(x3)
x3 = gru(x3)
x3 = GlobalMaxPool1D()(x3)
x3 = dense_2(x3)
_norm = Lambda(lambda x: K.l2_normalize(x, axis=-1))
x1 = _norm(x1)
x2 = _norm(x2)
x3 = _norm(x3)
x = Concatenate(axis=-1)([x1, x2, x3])
model = Model([input_1, input_2, input_3], x)
model.compile(loss=triplet_loss, optimizer=Adam(lr))
model.summary()
return model
def build(self, input_shape):
self.filters_in = input_shape[-1]
self.filters = max(1, int(self.filters_in * self.se_ratio))
if self.pool_mode == 'avg':
self.pool = GlobalAvgPool2D(name=f'{self.name}_pool')
elif self.pool_mode == 'max':
self.pool = GlobalMaxPool2D(name=f'{self.name}_pool')
else:
raise NotImplementedError(f'Invalid pool={self.pool_mode}')
self.reshape = Reshape((1, 1, self.filters_in),
name=f'{self.name}_reshape')
self.conv1 = Conv2D(self.filters,
1,
activation=self.activation,
name=f'{self.name}_conv',
**self.conv_kw)
self.conv2 = Conv2D(self.filters_in,
1,
activation='sigmoid',
name=f'{self.name}_proj',
**self.conv_kw)
return super().build(input_shape)
def create_homographynet_vgg(weights_file=None, input_shape=(128, 128, 2), num_pts=4, pooling=None, **kwargs):
"""
Use fully convolutional structure to support any input shape
"""
model = Sequential(name='homographynet')
model.add(InputLayer(input_shape, name='input_1'))
#model.add(InputLayer((None, None, 2), name='input_1')) # support any input shape
# 4 Layers with 64 filters, then another 4 with 128 filters
filters = 4 * [64] + 4 * [128]
for i, f in enumerate(filters, start=1):
model.add(Conv2D(filters=f, kernel_size=3, padding='same',
activation='relu', name='conv2d_{}'.format(i)))
model.add(BatchNormalization(axis=-1, name='batch_normalization_{}'.format(i)))
# MaxPooling after every 2 Conv layers except the last one
if i % 2 == 0 and i != 8:
model.add(MaxPooling2D(pool_size=(2,2), strides=(2, 2),
name='max_pooling2d_{}'.format(int(i/2))))
if pooling == 'max':
model.add(GlobalMaxPool2D(name='global_max_pool2d')) # to support any input shape
elif pooling == 'avg':
model.add(GlobalAveragePooling2D(name='global_avg_pool2d')) # to support any input shape
elif pooling == 'flatten':
model.add(Flatten(name='flatten_1'))
model.add(Dropout(rate=0.5, name='dropout_1'))
else:
raise ValueError
model.add(Dense(units=1024, activation='relu', name='dense_1'))
model.add(Dropout(rate=0.5, name='dropout_2'))
# Regression model
model.add(Dense(units=2*num_pts, name='dense_2')) # no activation
if weights_file is not None:
model.load_weights(weights_file)
return model
def image_model(lr=0.0001):
input_1 = Input(shape=(None, None, 3))
base_model = ResNet50(weights='imagenet', include_top=False)
x1 = base_model(input_1)
x1 = GlobalMaxPool2D()(x1)
dense_1 = Dense(vec_dim, activation="linear", name="dense_image_1")
x1 = dense_1(x1)
_norm = Lambda(lambda x: K.l2_normalize(x, axis=-1))
x1 = _norm(x1)
model = Model([input_1], x1)
model.compile(loss="mae", optimizer=Adam(lr))
model.summary()
return model
def build_2d_model(input_shape=input_shape, nclass=nclass):
input_wave = Input(shape=input_shape)
xception = Xception(input_shape=input_shape,
weights=None,
include_top=False)
x = xception(input_wave)
x = GlobalMaxPool2D()(x)
x = Dropout(rate=0.1)(x)
x = Dense(128, activation=activations.relu)(x)
x = Dense(nclass, activation=activations.softmax)(x)
model = models.Model(inputs=input_wave, outputs=x)
opt = optimizers.Adam()
model.compile(optimizer=opt,
loss=losses.sparse_categorical_crossentropy,
metrics=["acc"])
model.summary()
return model
def create_homographynet_mobilenet_v2(weights_file=None, input_shape=(128, 128, 2), num_pts=4, alpha=1.0,
pooling=None, **kwargs):
"""
Use fully convolutional structure to support any input shape
"""
base_model = mobilenet_v2.MobileNetV2(input_shape=input_shape, include_top=False, weights=None, alpha=alpha)
# The output shape just before the pooling and dense layers is: (4, 4, 1024)
x = base_model.output
if pooling == 'max':
x = GlobalMaxPool2D(name='global_max_pool2d')(x) # to support any input shape
elif pooling == 'avg':
x = GlobalAveragePooling2D(name='global_avg_pool2d')(x) # to support any input shape
else:
x = Flatten(name='flatten')(x)
x = Dropout(rate=0.5, name='dropout')(x)
x = Dense(2*num_pts, name='preds')(x)
model = Model(inputs=base_model.input, outputs=x, name='homographynet_mobilenet_v2')
if weights_file is not None:
model.load_weights(weights_file)
return model
def SqueezeNet(include_top=True,
weights="imagenet",
model_input=None,
non_top_pooling=None,
num_classes=1000,
model_path="",
initial_num_classes=None,
transfer_with_full_training=True):
if (weights == "imagenet" and num_classes != 1000):
raise ValueError(
"You must parse in SqueezeNet model trained on the 1000 class ImageNet"
)
image_input = model_input
network = Conv2D(64, (3, 3), strides=(2, 2), padding="valid")(image_input)
network = Activation("relu")(network)
network = MaxPool2D(pool_size=(3, 3), strides=(2, 2))(network)
network = squeezenet_fire_module(input=network,
input_channel_small=16,
input_channel_large=64)
network = squeezenet_fire_module(input=network,
input_channel_small=16,
input_channel_large=64)
network = MaxPool2D(pool_size=(3, 3), strides=(2, 2))(network)
network = squeezenet_fire_module(input=network,
input_channel_small=32,
input_channel_large=128)
network = squeezenet_fire_module(input=network,
input_channel_small=32,
input_channel_large=128)
network = MaxPool2D(pool_size=(3, 3), strides=(2, 2))(network)
network = squeezenet_fire_module(input=network,
input_channel_small=48,
input_channel_large=192)
network = squeezenet_fire_module(input=network,
input_channel_small=48,
input_channel_large=192)
network = squeezenet_fire_module(input=network,
input_channel_small=64,
input_channel_large=256)
network = squeezenet_fire_module(input=network,
input_channel_small=64,
input_channel_large=256)
if (include_top):
network = Dropout(0.5)(network)
if (initial_num_classes != None):
network = Conv2D(initial_num_classes,
kernel_size=(1, 1),
padding="valid",
name="last_conv")(network)
else:
network = Conv2D(num_classes,
kernel_size=(1, 1),
padding="valid",
name="last_conv")(network)
network = Activation("relu")(network)
network = GlobalAvgPool2D()(network)
network = Activation("softmax")(network)
else:
if (non_top_pooling == "Average"):
network = GlobalAvgPool2D()(network)
elif (non_top_pooling == "Maximum"):
network = GlobalMaxPool2D()(network)
elif (non_top_pooling == None):
pass
input_image = image_input
model = Model(inputs=input_image, outputs=network)
if (weights == "imagenet"):
weights_path = model_path
model.load_weights(weights_path)
return model
elif (weights == "trained"):
weights_path = model_path
model.load_weights(weights_path)
return model
elif (weights == "continued"):
weights_path = model_path
model.load_weights(weights_path)
return model
elif (weights == "transfer"):
weights_path = model_path
model.load_weights(weights_path)
if (transfer_with_full_training == False):
for eachlayer in model.layers:
eachlayer.trainable = False
print("Training with top layers of the Model")
else:
print("Training with all layers of the Model")
network2 = model.layers[-5].output
network2 = Conv2D(num_classes,
kernel_size=(1, 1),
padding="valid",
name="last_conv")(network2)
network2 = Activation("relu")(network2)
network2 = GlobalAvgPool2D()(network2)
network2 = Activation("softmax")(network2)
new_model = Model(inputs=model.input, outputs=network2)
return new_model
elif (weights == "custom"):
return model
def baseline_model():
input_1 = Input(shape=(IMG_SIZE_FN[0], IMG_SIZE_FN[1], 3))
input_2 = Input(shape=(IMG_SIZE_FN[0], IMG_SIZE_FN[1], 3))
input_3 = Input(shape=(IMG_SIZE_VGG[0], IMG_SIZE_VGG[1], 3))
input_4 = Input(shape=(IMG_SIZE_VGG[0], IMG_SIZE_VGG[1], 3))
model_vgg = VGGFace(model='resnet50', include_top=False)
model_facenet = FaceNet()
x1 = model_facenet(input_1)
x2 = model_facenet(input_2)
x3 = model_vgg(input_3) # (1, 2048)
x4 = model_vgg(input_4) # (1, 2048)
x1 = Reshape((1, 1, 128))(x1)
x2 = Reshape((1, 1, 128))(x2)
x1 = Concatenate(axis=-1)([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)]) # (1, 256)
x2 = Concatenate(axis=-1)([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)]) # (1, 256)
##################### Combination #7 FIW ###############
# x1_x2_sum = Add()([x1, x2]) # x1 + x2
# x1_x2_diff = Subtract()([x1, x2]) # x1 - x2
# x1_x2_product = Multiply()([x1, x2]) # x1 * x2
#
# x1_sqrt = Lambda(lambda tensor: signed_sqrt(tensor))(x1)
# x2_sqrt = Lambda(lambda tensor: signed_sqrt(tensor))(x2)
#
# x1_sqrt_x2_sqrt_sum = Add()([x1_sqrt, x2_sqrt])
#
# x1_square = Multiply()([x1, x1])
# x2_square = Multiply()([x2, x2])
# x1_sq_x2_sq_sum = Add()([x1_square, x2_square])
#
#
# x3_x4_sum = Add()([x3, x4]) # x3 + x4
# x3_x4_diff = Subtract()([x3, x4]) # x3 - x4
# x3_x4_product = Multiply()([x3, x4]) # x3 * x4
#
# x3_sqrt = Lambda(lambda tensor: signed_sqrt(tensor))(x3)
# x4_sqrt = Lambda(lambda tensor: signed_sqrt(tensor))(x4)
#
# x3_sqrt_x4_sqrt_sum = Add()([x3_sqrt, x4_sqrt])
#
# x3_square = Multiply()([x3, x3])
# x4_square = Multiply()([x4, x4])
# x3_sq_x4_sq_sum = Add()([x3_square, x4_square])
#
#
#
# x3_x4_sum = Conv2D(128, [1, 1])(x3_x4_sum) # (1, 128)
# x3_x4_diff = Conv2D(128, [1, 1])(x3_x4_diff) # (1, 128)
# x3_x4_product = Conv2D(128, [1, 1])(x3_x4_product) # (1, 128)
# x3_sqrt_x4_sqrt_sum = Conv2D(128, [1, 1])(x3_sqrt_x4_sqrt_sum)
# x3_sq_x4_sq_sum = Conv2D(128, [1, 1])(x3_sq_x4_sq_sum)
#
# x = Concatenate(axis=-1)(
# [Flatten()(x3_x4_sum), x1_x2_sum,
# Flatten()(x3_x4_diff), x1_x2_diff,
# Flatten()(x3_x4_product), x1_x2_product,
# Flatten()(x3_sqrt_x4_sqrt_sum), x1_sqrt_x2_sqrt_sum,
# Flatten()(x3_sq_x4_sq_sum), x1_sq_x2_sq_sum
# ])
########################################################
################### Combination #6 FIW ##################
# x1_square = Multiply()([x1, x1])
# x2_square = Multiply()([x2, x2])
# x1_sq_x2_sq_diff = Subtract()([x1_square, x2_square])
#
# x1_x2_diff = Subtract()([x1, x2])
# x1_x2_diff_sq = Multiply()([x1_x2_diff, x1_x2_diff])
#
# x1_x2_product = Multiply()([x1, x2])
#
# x1_x2_sum = Add()([x1, x2]) # x1 + x2
#
# x1_sqrt = Lambda(lambda tensor: signed_sqrt(tensor))(x1)
# x2_sqrt = Lambda(lambda tensor: signed_sqrt(tensor))(x2)
#
# x1_sqrt_x2_sqrt_diff = Subtract()([x1_sqrt, x2_sqrt])
# x1_sqrt_x2_sqrt_sum = Add()([x1_sqrt, x2_sqrt])
#
#
# x3_square = Multiply()([x3, x3])
# x4_square = Multiply()([x4, x4])
# x3_sq_x4_sq_diff = Subtract()([x3_square, x4_square])
#
# x3_x4_diff = Subtract()([x3, x4])
# x3_x4_diff_sq = Multiply()([x3_x4_diff, x3_x4_diff])
#
# x3_x4_product = Multiply()([x3, x4])
#
# x3_x4_sum = Add()([x3, x4]) # x1 + x2
#
# x3_sqrt = Lambda(lambda tensor: signed_sqrt(tensor))(x3)
# x4_sqrt = Lambda(lambda tensor: signed_sqrt(tensor))(x4)
#
# x3_sqrt_x4_sqrt_diff = Subtract()([x3_sqrt, x4_sqrt])
# x3_sqrt_x4_sqrt_sum = Add()([x3_sqrt, x4_sqrt])
#
# x3_sq_x4_sq_diff = Conv2D(128, [1, 1])(x3_sq_x4_sq_diff)
# x3_x4_diff_sq = Conv2D(128, [1, 1])(x3_x4_diff_sq)
# x3_x4_product = Conv2D(128, [1, 1])(x3_x4_product)
# x3_x4_sum = Conv2D(128, [1, 1])(x3_x4_sum)
# x3_x4_diff = Conv2D(128, [1, 1])(x3_x4_diff)
# x3_sqrt_x4_sqrt_diff = Conv2D(128, [1, 1])(x3_sqrt_x4_sqrt_diff)
# x3_sqrt_x4_sqrt_sum = Conv2D(128, [1, 1])(x3_sqrt_x4_sqrt_sum)
#
#
# x = Concatenate(axis=-1)([
# Flatten()(x3_sq_x4_sq_diff), x1_sq_x2_sq_diff,
# Flatten()(x3_x4_diff_sq), x1_x2_diff_sq,
# Flatten()(x3_x4_product), x1_x2_product,
# Flatten()(x3_x4_sum), x1_x2_sum,
# Flatten()(x3_x4_diff), x1_x2_diff,
# Flatten()(x3_sqrt_x4_sqrt_sum), x1_sqrt_x2_sqrt_sum,
# Flatten()(x3_sqrt_x4_sqrt_diff), x1_sqrt_x2_sqrt_diff
# ])
#################### Combination #5 FIW ################
# x3 = Conv2D(128, [1, 1])(x3)
# x4 = Conv2D(128, [1, 1])(x4)
#
#
# x = Concatenate(axis=-1)(
# [Flatten()(x4), x2,
# Flatten()(x3), x1])
########################################################
################### Combination #4 FIW ##################
# x1_x2_sum = Add()([x1, x2]) # x1 + x2
# x1_x2_diff = Subtract()([x1, x2]) # x1 - x2
#
#
# x3_x4_sum = Add()([x3, x4]) # x3 + x4
# x3_x4_diff = Subtract()([x3, x4]) # x3 - x4
#
#
# x3_x4_sum = Conv2D(128, [1, 1])(x3_x4_sum) # (1, 128)
# x3_x4_diff = Conv2D(128, [1, 1])(x3_x4_diff) # (1, 128)
#
#
# x = Concatenate(axis=-1)(
# [Flatten()(x3_x4_sum), x1_x2_sum,
# Flatten()(x3_x4_diff), x1_x2_diff])
###########################################################
############# Combination #3 FIW ################
x1_x2_sum = Add()([x1, x2]) # x1 + x2
x1_x2_diff = Subtract()([x1, x2]) # x1 - x2
x1_x2_product = Multiply()([x1, x2]) # x1 * x2
x3_x4_sum = Add()([x3, x4]) # x3 + x4
x3_x4_diff = Subtract()([x3, x4]) # x3 - x4
x3_x4_product = Multiply()([x3, x4]) # x3 * x4
x3_x4_sum = Conv2D(128, [1, 1])(x3_x4_sum) # (1, 128)
x3_x4_diff = Conv2D(128, [1, 1])(x3_x4_diff) # (1, 128)
x3_x4_product = Conv2D(128, [1, 1])(x3_x4_product) # (1, 128)
x = Concatenate(axis=-1)(
[Flatten()(x3_x4_sum), x1_x2_sum,
Flatten()(x3_x4_diff), x1_x2_diff,
Flatten()(x3_x4_product), x1_x2_product])
##########################################
################## Combination #2 FIW ########################
# x1_square = Multiply()([x1, x1])
# x2_square = Multiply()([x2, x2])
# x1_sq_x2_sq_diff = Subtract()([x1_square, x2_square])
#
# x1_x2_diff = Subtract()([x1, x2])
# x1_x2_diff_sq = Multiply()([x1_x2_diff, x1_x2_diff])
#
#
# x3_square = Multiply()([x3, x3])
# x4_square = Multiply()([x4, x4])
# x3_sq_x4_sq_diff = Subtract()([x3_square, x4_square])
#
# x3_x4_diff = Subtract()([x3, x4])
# x3_x4_diff_sq = Multiply()([x3_x4_diff, x3_x4_diff])
#
#
# x3_sq_x4_sq_diff = Conv2D(128, [1, 1])(x3_sq_x4_sq_diff)
# x3_x4_diff_sq = Conv2D(128, [1, 1])(x3_x4_diff_sq)
#
# x = Concatenate(axis=-1)([
# Flatten()(x3_sq_x4_sq_diff), x1_sq_x2_sq_diff,
# Flatten()(x3_x4_diff_sq), x1_x2_diff_sq
# ])
################################################################
################## Combination #1 FIW ###################
# x1_square = Multiply()([x1, x1])
# x2_square = Multiply()([x2, x2])
# x1_sq_x2_sq_diff = Subtract()([x1_square, x2_square])
#
# x1_x2_diff = Subtract()([x1, x2])
# x1_x2_diff_sq = Multiply()([x1_x2_diff, x1_x2_diff])
#
# x1_x2_product = Multiply()([x1, x2])
#
# x3_square = Multiply()([x3, x3])
# x4_square = Multiply()([x4, x4])
# x3_sq_x4_sq_diff = Subtract()([x3_square, x4_square])
#
# x3_x4_diff = Subtract()([x3, x4])
# x3_x4_diff_sq = Multiply()([x3_x4_diff, x3_x4_diff])
#
# x3_x4_product = Multiply()([x3, x4])
# x3_sq_x4_sq_diff = Conv2D(128, [1, 1])(x3_sq_x4_sq_diff)
# x3_x4_diff_sq = Conv2D(128, [1, 1])(x3_x4_diff_sq)
# x3_x4_product = Conv2D(128, [1, 1])(x3_x4_product)
# x = Concatenate(axis=-1)([
# Flatten()(x3_sq_x4_sq_diff), x1_sq_x2_sq_diff,
# Flatten()(x3_x4_diff_sq), x1_x2_diff_sq,
# Flatten()(x3_x4_product), x1_x2_product
# ])
#############################################
######## Combination 0 #################
# x = Dense(128, activation="relu")(x)
# x = Dropout(0.02)(x)
# out = Dense(1, activation="sigmoid")(x)
######## Combination 1 #################
# x = Dense(64, activation="relu")(x)
# x = Dropout(0.02)(x)
# out = Dense(1, activation="sigmoid")(x)
######### Combination 2 ################
# x = Dense(128, activation="relu")(x)
# x = Dropout(0.02)(x)
# x = Dense(64, activation="relu")(x)
# x = Dropout(0.02)(x)
# out = Dense(1, activation="sigmoid")(x)
######### Combination 3 ################
# x = Dense(256, activation="relu")(x)
# x = Dropout(0.02)(x)
# x = Dense(128, activation="relu")(x)
# x = Dropout(0.02)(x)
# x = Dense(64, activation="relu")(x)
# x = Dropout(0.02)(x)
# out = Dense(1, activation="sigmoid")(x)
######### Combination 4 ################
# x = Dense(512, activation="relu")(x)
# x = Dropout(0.02)(x)
# x = Dense(256, activation="relu")(x)
# x = Dropout(0.02)(x)
# x = Dense(128, activation="relu")(x)
# x = Dropout(0.02)(x)
# x = Dense(64, activation="relu")(x)
# x = Dropout(0.02)(x)
# out = Dense(1, activation="sigmoid")(x)
######### Combination 5-8 ################
#
x = Dense(1024, activation="relu")(x)
x = Dropout(0.02)(x)
x = Dense(512, activation="relu")(x)
x = Dropout(0.02)(x)
x = Dense(256, activation="relu")(x)
x = Dropout(0.02)(x)
x = Dense(128, activation="relu")(x)
x = Dropout(0.02)(x)
x = Dense(64, activation="relu")(x)
x = Dropout(0.02)(x)
out = Dense(1, activation="sigmoid")(x)
model = Model([input_1, input_2, input_3, input_4], out)
# input_1 = Input(shape=(IMG_SIZE_VGG[0], IMG_SIZE_VGG[1], 3))
# input_2 = Input(shape=(IMG_SIZE_VGG[0], IMG_SIZE_VGG[1], 3))
# base_model = VGGFace(model='resnet50', include_top=False)
# # base_model.trainable = False
# for x in base_model.layers[:-3]:
# x.trainable = True
# x1 = base_model(input_1)
# x2 = base_model(input_2)
#
#
# # x1 = GlobalMaxPool2D()(x1)
# # x2 = GlobalAvgPool2D()(x2)
#
# x3 = Subtract()([x1, x2])
# x3 = Multiply()([x3, x3])
#
# x1_ = Multiply()([x1, x1])
# x2_ = Multiply()([x2, x2])
# x4 = Subtract()([x1_, x2_])
#
# x5 = Multiply()([x1, x2])
#
# x = Concatenate(axis=-1)([x3, x4, x5])
# # # # x = Dense(512, activation="relu")(x)
# # # # x = Dropout(0.03)(x)
# x = Dense(128, activation="relu")(x)
# x = Dropout(0.02)(x)
# out = Dense(1, activation="sigmoid")(x)
#
# model = Model([input_1, input_2], out)
#
model.compile(loss="binary_crossentropy", metrics=['acc', auc], optimizer=Adam(0.00001))
#
model.summary()
return model
def create_model(input_shape, l1fc, l1fs, l1st, l2fc, l2fs, l2st, l3fc, l3fs, eac_size, res_c, res_fc, res_fs):
inp = Input(shape=(input_shape[0], input_shape[1], 3))
c1 = Conv2D(l1fc, kernel_size=l1fs, strides=l1st, use_bias=True, padding="same",
data_format="channels_last", kernel_initializer=laplacian_group_initializer)(inp)
l1 = LeakyReLU(alpha=0.2)(c1)
eac1_obj = EdgeAndCenterExtractionLayer(width=eac_size)
eac1 = eac1_obj(l1)
eac1.set_shape(eac1_obj.compute_output_shape(l1.shape))
c2 = Conv2D(l2fc, kernel_size=l2fs, strides=l2st, use_bias=True, padding="same",
data_format="channels_last")(l1)
l2 = LeakyReLU(alpha=0.2)(c2)
eac2_obj = EdgeAndCenterExtractionLayer(width=eac_size)
eac2 = eac2_obj(l2)
eac2.set_shape(eac2_obj.compute_output_shape(l2.shape))
c3 = Conv2D(l3fc, kernel_size=l3fs, strides=1, use_bias=True, padding="same",
data_format="channels_last")(l2)
last_layer = c3
prev_layer = None
for i in range(res_c):
res_act = LeakyReLU(alpha=0.2)(last_layer)
if prev_layer is not None:
res_act = Add()([res_act, prev_layer])
prev_layer = last_layer
last_layer = Conv2D(res_fc, kernel_size=res_fs, strides=1, use_bias=True, padding="same",
data_format="channels_last")(res_act)
eac3_obj = EdgeAndCenterExtractionLayer(width=eac_size)
eac3 = eac3_obj(c3)
eac3.set_shape(eac3_obj.compute_output_shape(c3.shape))
eac3_max_grid = MaxPool2D((eac_size, eac_size), strides=eac_size,
padding="valid", data_format="channels_last")(eac3)
eac3_avg_grid = AveragePooling2D((eac_size, eac_size), strides=eac_size,
padding="valid", data_format="channels_last")(eac3)
features = [GlobalVarianceLayer()(c1),
GlobalVarianceLayer()(c2),
GlobalVarianceLayer()(c3),
GlobalMaxPool2D(data_format="channels_last")(c1),
GlobalMaxPool2D(data_format="channels_last")(c2),
GlobalMaxPool2D(data_format="channels_last")(c3),
GlobalAveragePooling2D(data_format="channels_last")(c1),
GlobalAveragePooling2D(data_format="channels_last")(c2),
GlobalAveragePooling2D(data_format="channels_last")(c3),
GlobalMaxPool2D(data_format="channels_last")(eac1),
GlobalMaxPool2D(data_format="channels_last")(eac2),
GlobalMaxPool2D(data_format="channels_last")(eac3),
GlobalAveragePooling2D(data_format="channels_last")(eac1),
GlobalAveragePooling2D(data_format="channels_last")(eac2),
GlobalAveragePooling2D(data_format="channels_last")(eac3),
GlobalVarianceLayer()(eac1),
GlobalVarianceLayer()(eac2),
GlobalVarianceLayer()(eac3),
Flatten()(VarianceLayer((eac_size, eac_size))(eac1)),
Flatten()(VarianceLayer((eac_size, eac_size))(eac2)),
Flatten()(VarianceLayer((eac_size, eac_size))(eac3)),
GlobalVarianceLayer()(eac3_max_grid),
GlobalVarianceLayer()(eac3_avg_grid),
Flatten()(eac3_max_grid),
Flatten()(eac3_avg_grid)
]
if res_c > 0:
res_eac = EdgeAndCenterExtractionLayer(width=eac_size)(last_layer)
features.append(GlobalVarianceLayer()(last_layer))
features.append(GlobalMaxPool2D()(last_layer))
features.append(GlobalAveragePooling2D()(last_layer))
features.append(GlobalVarianceLayer()(res_eac))
features.append(GlobalMaxPool2D()(res_eac))
features.append(GlobalAveragePooling2D()(res_eac))
features.append(Flatten()(VarianceLayer((eac_size, eac_size))(res_eac)))
res_eac_max_grid = MaxPool2D((eac_size, eac_size), strides=eac_size,
padding="valid", data_format="channels_last")(res_eac)
res_eac_avg_grid = AveragePooling2D((eac_size, eac_size), strides=eac_size,
padding="valid", data_format="channels_last")(res_eac)
features.append(GlobalVarianceLayer()(res_eac_max_grid))
features.append(GlobalVarianceLayer()(res_eac_avg_grid))
features.append(Flatten()(res_eac_max_grid))
features.append(Flatten()(res_eac_avg_grid))
feature_vector = Concatenate()(features)
o = Dense(2, activation="softmax", use_bias=True, name="output")(feature_vector)
return Model(inputs=inp, outputs=o)
activation=activation)(input_data)
if bn:
x = BatchNormalization()(x)
if pool:
x = MaxPool2D()(x)
return x
input_layer = Input(shape=(IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS))
x = conv_block(input_layer, filters=32, bn=False, pool=False)
x = conv_block(x, filters=32 * 2)
x = conv_block(x, filters=32 * 3)
x = conv_block(x, filters=32 * 4)
x = conv_block(x, filters=32 * 5)
x = conv_block(x, filters=32 * 6)
bottleneck = GlobalMaxPool2D()(x)
# for age calculation
age_x = Dense(128, activation='relu')(bottleneck)
age_output = Dense(1, activation='sigmoid', name='age_output')(age_x)
# for race prediction
race_x = Dense(128, activation='relu')(bottleneck)
race_output = Dense(len(RACE_ID_MAP), activation='softmax',
name='race_output')(race_x)
# for gender prediction
gender_x = Dense(128, activation='relu')(bottleneck)
gender_output = Dense(len(GENDER_ID_MAP),
activation='softmax',
name='gender_output')(gender_x)
def __init__(self,
model=None,
loss=-1,
useRestnet=True,
inputShape=(3, 300, 600, 3),
gridCount=243,
modelOptimizer=tf.keras.optimizers.Adam()):
'''
The function is used to load or initialize a new model
useRestnet : set to True to use pretrained frozen restnet model
set to False to use trainable CNN model
inputShape: Shape of input image set
(<numer-of-images>, <image-width>, <image-height>, <RGB-values>)
gridCount: Number of ouput grids to predict on
'''
if model == None:
convnet = tf.keras.Sequential()
if useRestnet:
# use restnet CNN
restnet = ResNet50(include_top=False,
weights='imagenet',
input_shape=inputShape[1:])
# Freeze model
restnet.trainable = False
convnet.add(restnet)
else:
# Use trainable CNN
convnet.add(
Conv2D(128, (3, 3),
input_shape=inputShape[1:],
padding='same',
activation='relu'))
convnet.add(
Conv2D(128, (3, 3), padding='same', activation='relu'))
convnet.add(BatchNormalization(momentum=.6))
convnet.add(MaxPool2D())
convnet.add(
Conv2D(64, (3, 3), padding='same', activation='relu'))
convnet.add(
Conv2D(64, (3, 3), padding='same', activation='relu'))
convnet.add(BatchNormalization(momentum=.6))
convnet.add(MaxPool2D())
convnet.add(
Conv2D(64, (3, 3), padding='same', activation='relu'))
convnet.add(
Conv2D(64, (3, 3), padding='same', activation='relu'))
convnet.add(BatchNormalization(momentum=.6))
convnet.add(MaxPool2D())
convnet.add(
Conv2D(512, (3, 3), padding='same', activation='relu'))
convnet.add(
Conv2D(512, (3, 3), padding='same', activation='relu'))
convnet.add(BatchNormalization(momentum=.6))
convnet.add(GlobalMaxPool2D())
self.model = tf.keras.Sequential()
# Connect the CNN to an LSTM
self.model.add(TimeDistributed(convnet, input_shape=inputShape))
self.model.add(LSTM(64))
self.model.add(Dense(1024, activation='relu'))
self.model.add(Dropout(.5))
self.model.add(Dense(512, activation='relu'))
self.model.add(Dropout(.5))
self.model.add(Dense(128, activation='relu'))
self.model.add(Dropout(.5))
self.model.add(Dense(64, activation='relu'))
self.model.add(Dense(gridCount, activation='softmax'))
self.model.compile(loss=tf.keras.losses.categorical_crossentropy,
optimizer=modelOptimizer,
metrics=['categorical_accuracy'])
else:
# load pre trained model
self.model = model
self.model.summary()
self.loss = loss
def get_model(bn_momentum):
pt_cloud = Input(shape=(None, 5), dtype=tf.float32,
name='pt_cloud') # BxNx3
# Input transformer (B x N x 3 -> B x N x 3)
#pt_cloud_transform = TNet(bn_momentum=bn_momentum)(pt_cloud)
# Embed to 64-dim space (B x N x 3 -> B x N x 64)
pt_cloud_transform = tf.expand_dims(pt_cloud,
axis=2) # for weight-sharing of conv
# Stage1
x = CustomConv(64, (1, 1),
strides=(1, 1),
activation=tf.nn.relu,
apply_bn=True,
bn_momentum=bn_momentum)(pt_cloud_transform)
x = BatchNormalization()(x)
x = tf.nn.relu(x)
x = MaxPool2D(pool_size=(2, 1))(x)
#Stage2
f = [64, 64, 256]
x = ResidualConvBlock(filters=f, kernel_size=2)(x)
x = ResidualIdentityBlock(filters=f, kernel_size=2)(x)
#x = ResidualIdentityBlock(filters=f, kernel_size=2)(x)
#Stage3
f = [128, 128, 512]
x = ResidualConvBlock(filters=f, kernel_size=2)(x)
x = ResidualIdentityBlock(filters=f, kernel_size=2)(x)
#x = ResidualIdentityBlock(filters=f, kernel_size=2)(x)
#Stage4
f = [256, 256, 1024]
x = ResidualConvBlock(filters=f, kernel_size=2)(x)
x = ResidualIdentityBlock(filters=f, kernel_size=2)(x)
#x = ResidualIdentityBlock(filters=f, kernel_size=2)(x)
#Stage5
#f = [512, 512, 2048]
f = [512, 512, 1024]
x = ResidualConvBlock(filters=f, kernel_size=2)(x)
x = ResidualIdentityBlock(filters=f, kernel_size=2)(x)
x = ResidualIdentityBlock(filters=f, kernel_size=2)(x)
# hidden_64 = CustomConv(64, (1, 1), strides=(1, 1), activation=tf.nn.relu, apply_bn=True,
# bn_momentum=bn_momentum)(pt_cloud_transform)
# embed_64 = CustomConv(64, (1, 1), strides=(1, 1), activation=tf.nn.relu, apply_bn=True,
# bn_momentum=bn_momentum)(hidden_64)
#embed_64 = tf.squeeze(embed_64, axis=2)
# Feature transformer (B x N x 64 -> B x N x 64)
#embed_64_transform = TNet(bn_momentum=bn_momentum, add_regularization=True)(embed_64)
# Embed to 1024-dim space (B x N x 64 -> B x N x 1024)
#embed_64_transform = tf.expand_dims(embed_64, axis=2)
# hidden_64 = CustomConv(64, (1, 1), strides=(1, 1), activation=tf.nn.relu, apply_bn=True,
# bn_momentum=bn_momentum)(embed_64)
# hidden_128 = CustomConv(128, (1, 1), strides=(1, 1), activation=tf.nn.relu, apply_bn=True,
# bn_momentum=bn_momentum)(res2)
# embed_1024 = CustomConv(1024, (1, 1), strides=(1, 1), activation=tf.nn.relu, apply_bn=True,
# bn_momentum=bn_momentum)(hidden_128)
# x = tf.squeeze(x, axis=2)
# Global feature vector (B x N x 1024 -> B x 1024)
# global_descriptor = tf.reduce_max(x, axis=1)
global_descriptor = GlobalMaxPool2D()(x)
# FC layers to output k scores (B x 1024 -> B x 5)
# hidden_512 = CustomDense(512, activation=tf.nn.relu, apply_bn=True,
# bn_momentum=bn_momentum)(global_descriptor)
# hidden_512 = Dropout(rate=0.2)(hidden_512)
# hidden_256 = CustomDense(256, activation=tf.nn.relu, apply_bn=True,
# bn_momentum=bn_momentum)(hidden_512)
# hidden_256 = Dropout(rate=0.2)(hidden_256)
# logits = CustomDense(5, apply_bn=False)(hidden_256)
logits = Dense(units=5)(global_descriptor)
return Model(inputs=pt_cloud, outputs=logits)
# 定义函数式模型
# 定义模型输入，shape-(batch, 202)
sequence_input = Input(shape=(max_length, ))
# Embedding层，30000表示30000个词，每个词对应的向量为128维
embedding_layer = Embedding(input_dim=30000, output_dim=embedding_dims)
# embedded_sequences的shape-(batch, 202, 128)
embedded_sequences = embedding_layer(sequence_input)
# embedded_sequences的shape变成了(batch, 202, 128, 1)
embedded_sequences = K.expand_dims(embedded_sequences, axis=-1)
# 卷积核大小为3,列数必须等于词向量长度
cnn1 = Conv2D(filters=filters,
kernel_size=(3, embedding_dims),
activation='relu')(embedded_sequences)
cnn1 = GlobalMaxPool2D()(cnn1)
# 卷积核大小为4,列数必须等于词向量长度
cnn2 = Conv2D(filters=filters,
kernel_size=(4, embedding_dims),
activation='relu')(embedded_sequences)
cnn2 = GlobalMaxPool2D()(cnn2)
# 卷积核大小为5,列数必须等于词向量长度
cnn3 = Conv2D(filters=filters,
kernel_size=(5, embedding_dims),
activation='relu')(embedded_sequences)
cnn3 = GlobalMaxPool2D()(cnn3)
# 合并
merge = concatenate([cnn1, cnn2, cnn3], axis=-1)