def get_model_2():
'''
:return: возвращает арзитектуру сети для Yolo
'''
input_image = layers.Input(shape=(IMAGE_H/striding ,IMAGE_W/striding ,IMAGE_D/striding ,1))
x = layers.Conv3D(filters = 16 ,kernel_size=(5 ,5 ,5) ,strides = (1 ,1 ,1) ,padding = 'same', name = 'conv_0_1') \
(input_image)
x = layers.Conv3D(filters = 16 ,kernel_size=(5 ,5 ,5) ,strides = (1 ,1 ,1) ,padding = 'same', name = 'conv_0_2') \
(x)
x = layers.BatchNormalization(name='norm_0_1')(x)
x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)
x = layers.MaxPool3D(pool_size=(2 ,2 ,1))(x)
x = layers.Conv3D(filters = 32 ,kernel_size=(5 ,5 ,5) ,strides = (1 ,1 ,1) ,padding = 'same' ,name = 'conv_1_1')(x)
x = layers.Conv3D(filters = 32 ,kernel_size=(5 ,5 ,5) ,strides = (1 ,1 ,1) ,padding = 'same' ,name = 'conv_1_2')(x)
x = layers.BatchNormalization(name='norm_1_1')(x)
x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)
x = layers.MaxPool3D(pool_size=(4 ,4 ,4))(x)
x = layers.Conv3D(filters = 32 ,kernel_size=(5 ,5 ,5) ,strides = (1 ,1 ,1) ,padding = 'same', name = 'conv_2_1')(x)
x = layers.Conv3D(filters = 32 ,kernel_size=(5 ,5 ,5) ,strides = (1 ,1 ,1) ,padding = 'same', name = 'conv_2_2')(x)
x = layers.BatchNormalization(name='norm_5_1')(x)
x = layers.advanced_activations.LeakyReLU(alpha=ALPHA)(x)
x = layers.MaxPool3D(pool_size=(4 ,2 ,2))(x)
x = layers.Conv3D(filters= 32, kernel_size=(1 ,1 ,1), strides =(1 ,1 ,1), padding = 'same', name= 'pred_yolo')(x)
x = layers.Conv3D(filters= (2*N_DIM+1+num_classes )*num_boxes, kernel_size=(1 ,1 ,1), strides =(1 ,1 ,1), padding = 'same', name= 'yolo')(x)
output = layers.Reshape((GRID_H ,GRID_W ,GRID_D ,num_boxes, 2*N_DIM + 1 + num_classes))(x)
model = models.Model(input_image, output)
return model
def get_model(width=64, height=64 ,depth=16):
"""Build a 3D convolutional neural network model."""
inputs = keras.Input((depth,width, height, 3))
x = layers.Conv3D(filters=64, kernel_size=3, activation="relu",padding = 'same')(inputs)
x = layers.MaxPool3D(pool_size=2)(x)
x = layers.BatchNormalization()(x)
x = layers.Conv3D(filters=64, kernel_size=3, activation="relu",padding = 'same')(x)
x = layers.MaxPool3D(pool_size=2)(x)
x = layers.BatchNormalization()(x)
x = layers.Conv3D(filters=128, kernel_size=3, activation="relu",padding = 'same')(x)
x = layers.MaxPool3D(pool_size=2)(x)
x = layers.BatchNormalization()(x)
x = layers.Conv3D(filters=256, kernel_size=3, activation="relu",padding = 'same')(x)
x = layers.MaxPool3D(pool_size=2)(x)
x = layers.BatchNormalization()(x)
x = layers.GlobalAveragePooling3D()(x)
x = layers.Dense(units=512, activation="relu")(x)
x = layers.Dropout(0.3)(x)
outputs = layers.Dense(units=27, activation="softmax")(x)
# Define the model.
model = keras.Model(inputs, outputs, name="3dcnn")
return model
def create_model(inp_shape):
inp = layers.Input(shape=inp_shape)
x = layers.Conv3D(64, kernel_size=(7, 7), strides=(2, 2),
padding='same')(inp)
x = add_common_layers(x)
x = layers.MaxPool3D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
for i in range(3):
project_shortcut = True if i == 0 else False
x = residual_block(x, 128, 256, project_shortcut=project_shortcut)
for i in range(4):
# down-sampling is performed by conv3_1, conv4_1, and conv5_1 with a stride of 2
strides = (2, 2) if i == 0 else (1, 1)
x = residual_block(x, 256, 512, strides=strides)
for i in range(6):
strides = (2, 2) if i == 0 else (1, 1)
x = residual_block(x, 512, 1024, strides=strides)
for i in range(3):
strides = (2, 2) if i == 0 else (1, 1)
x = residual_block(x, 1024, 2048, strides=strides)
x = layers.GlobalAveragePooling3D()(x)
x = layers.Dense(16)(x)
x = Sparse(np.random.randint(16, size=10).reshape((16, 2)))(x)
x = layers.Dense(1)(x)
model = models.Model(inputs=[inp], outputs=[x])
print(model.summary())
return model
def add_activation_and_max_pooling(adjustable, model, use_batch_norm, batch_norm_name, first_layer=False):
"""One-liner for adding: pooling + activation + batchnorm
:param model: the model to add to
:return: the model with added activation and max pooling
:param trainable: boolean indicating if layer is trainable
"""
if adjustable.video_head_type == '3d_convolution':
if first_layer:
if adjustable.pooling_type == 'avg_pooling':
model.add(layers.AveragePooling3D(pool_size=(adjustable.pooling_size[0][0], adjustable.pooling_size[0][1], adjustable.pooling_size[0][2])))
else: # max_pooling
model.add(layers.MaxPool3D(pool_size=(adjustable.pooling_size[0][0], adjustable.pooling_size[0][1], adjustable.pooling_size[0][2])))
else:
if adjustable.pooling_type == 'avg_pooling':
model.add(layers.AveragePooling3D(pool_size=(adjustable.pooling_size[1][0], adjustable.pooling_size[1][1], adjustable.pooling_size[1][2])))
else: # max_pooling
model.add(layers.MaxPool3D(pool_size=(adjustable.pooling_size[1][0], adjustable.pooling_size[1][1], adjustable.pooling_size[1][2])))
model.add(layers.Activation(adjustable.activation_function))
if use_batch_norm:
model.add(layers.BatchNormalization(name=batch_norm_name, trainable=adjustable.trainable_bn))
return model
elif adjustable.video_head_type == 'cnn_lstm':
if first_layer:
if adjustable.pooling_type == 'avg_pooling':
pooling = layers.AveragePooling2D(
pool_size=(adjustable.pooling_size[0][0], adjustable.pooling_size[0][1]))(model)
else: # max_pooling
pooling = layers.MaxPool2D(
pool_size=(adjustable.pooling_size[0][0], adjustable.pooling_size[0][1]))(model)
else:
if adjustable.pooling_type == 'avg_pooling':
pooling = layers.AveragePooling2D(
pool_size=(adjustable.pooling_size[1][0], adjustable.pooling_size[1][1]))(model)
else: # max_pooling
pooling = layers.MaxPool2D(
pool_size=(adjustable.pooling_size[1][0], adjustable.pooling_size[1][1]))(model)
activation = layers.Activation(adjustable.activation_function)(pooling)
return activation
def soft_erode(img):
"""[This function performs soft-erosion operation on a float32 image]
Args:
img ([float32]): [image to be soft eroded]
Returns:
[float32]: [the eroded image]
"""
p1 = -KL.MaxPool3D(pool_size=(3, 3, 1),
strides=(1, 1, 1),
padding='same',
data_format=None)(-img)
p2 = -KL.MaxPool3D(pool_size=(3, 1, 3),
strides=(1, 1, 1),
padding='same',
data_format=None)(-img)
p3 = -KL.MaxPool3D(pool_size=(1, 3, 3),
strides=(1, 1, 1),
padding='same',
data_format=None)(-img)
return tf.math.minimum(tf.math.minimum(p1, p2), p3)
def soft_dilate(img):
"""[This function performs soft-dilation operation on a float32 image]
Args:
img ([float32]): [image to be soft dialated]
Returns:
[float32]: [the dialated image]
"""
return KL.MaxPool3D(pool_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
data_format=None)(img)
def create_autoencoder():
model = models.Sequential()
model.add(
layers.Conv3D(64,
5,
padding='same',
activation='relu',
input_shape=(80, 80, 64, 1)))
model.add(layers.MaxPool3D())
model.add(layers.Conv3D(128, 5, padding='same', activation='relu'))
model.add(layers.MaxPool3D())
model.add(layers.Conv3D(128, 5, padding='same', activation='relu'))
model.add(layers.MaxPool3D())
model.add(layers.Conv3D(256, 5, padding='same', activation='relu'))
model.add(layers.Deconv3D(256, 4, padding='same', activation='relu'))
model.add(layers.UpSampling3D())
model.add(layers.Deconv3D(128, 4, padding='same', activation='relu'))
model.add(layers.UpSampling3D())
model.add(layers.Deconv3D(128, 4, padding='same', activation='relu'))
model.add(layers.UpSampling3D())
model.add(layers.Deconv3D(64, 4, padding='same', activation='relu'))
model.add(layers.Deconv3D(1, 4, padding='same', activation='relu'))
return model
def conv2dlstm_model(window_size=32, n_notes=58):
notes_model = models.Sequential()
notes_model.add(layers.ConvLSTM2D(32, 3, return_sequences=True, padding='same',
input_shape=(None, window_size, None, 3)))
notes_model.add(layers.MaxPool3D(2, 2))
notes_model.add(layers.ConvLSTM2D(64, 3, padding='same'))
notes_model.add(layers.GlobalAveragePooling2D())
beats_input = layers.Input(shape=(16,))
features = layers.concatenate([notes_model.output, beats_input])
dropout1 = layers.Dropout(0.2)(features)
fc_1 = layers.Dense(100, activation='relu')(dropout1)
dropout2 = layers.Dropout(0.2)(fc_1)
output_notes = layers.Dense(n_notes + 1, activation='softmax')(dropout2)
output_slur = layers.Dense(1, activation='sigmoid')(fc_1)
return models.Model(inputs=[notes_model.input, beats_input], outputs=[output_notes, output_slur])
def build(self):
# Initialisation
input_layer = kl.Input(shape=self.input_shape, name="input_layer")
cur_layer = input_layer
levels = []
# Downward
for i in range(self.depth):
n_kernels = self.n_base_filters * (2**i)
conv_a = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"conv_a_{i}")(cur_layer)
conv_b = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"conv_b_{i}")(conv_a)
if i < self.depth - 1:
pool = kl.MaxPool3D(pool_size=(2, 2, 2),
padding="same",
name=f"pool_{i}")(conv_b)
cur_layer = pool
levels.append([conv_a, conv_b, pool])
else:
cur_layer = conv_b
levels.append([conv_a, conv_b])
# Upward
for i in range(self.depth - 2, -1, -1):
n_kernels = self.n_base_filters * (2**i)
up = kl.UpSampling3D(size=(2, 2, 2),
name=f"upsampling_{i}")(cur_layer)
upconv_a = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"upconv_a_{i}")(up)
merge = kl.Concatenate(axis=4,
name=f"merge{i}")([levels[i][1], upconv_a])
upconv_b = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"upconv_b_{i}")(merge)
upconv_c = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"upconv_c_{i}")(upconv_b)
cur_layer = upconv_c
# Finalisation
output_layer = kl.Conv3D(self.n_classes,
1,
activation="sigmoid",
name="output_layer")(cur_layer)
model = Model(inputs=input_layer, outputs=output_layer)
return model
def build_model(self):
chm_input = Input(shape=(20, 20, 1), name="chm")
rgb_input = Input(shape=(200, 200, 3), name="rgb")
hsi_input = Input(shape=(20, 20, 3), name="hsi")
las_input = Input(shape=(40, 40, 70, 1), name="las")
# RGB downsample network
rgb_down = layers.Conv2D(3, 5, activation="relu")(rgb_input)
rgb_down = layers.Conv2D(3, 5, activation="relu")(rgb_down)
rgb_down = layers.MaxPool2D(2)(rgb_down)
rgb_down = layers.Conv2D(8, 5, activation="relu")(rgb_down)
rgb_down = layers.Conv2D(8, 5, activation="relu")(rgb_down)
rgb_down = layers.MaxPool2D(2)(rgb_down)
rgb_down = layers.Conv2D(16, 4, activation="relu")(rgb_down)
rgb_down = layers.Conv2D(16, 4, activation="relu")(rgb_down)
rgb_down = layers.MaxPool2D(2, name="rgb_down")(rgb_down)
rgb_down = layers.Conv2D(32, 4, activation="relu")(rgb_down)
rgb_down = layers.Conv2D(32, 4, activation="relu")(rgb_down)
rgb_down = layers.Flatten()(rgb_down)
"""
# HSI upsample network
hsi_up = layers.Conv2D(3, 2, activation="relu", padding="same")(hsi_input)
hsi_up = layers.UpSampling2D(3)(hsi_up)
#hsi_up = layers.Dropout(0.4)(hsi_up)
hsi_up = layers.Conv2D(3, 4, activation="relu")(hsi_up)
hsi_up = layers.Conv2D(3, 4, activation="relu")(hsi_up)
hsi_up = layers.UpSampling2D(2)(hsi_up)
#hsi_up = layers.Dropout(0.4)(hsi_up)
hsi_up = layers.Conv2D(3, 5, activation="relu")(hsi_up)
hsi_up = layers.Conv2D(3, 5, activation="relu")(hsi_up)
hsi_up = layers.UpSampling2D(2, name="hsi_up")(hsi_up)
#hsi_up = layers.Dropout(0.4)(hsi_up)
# CHM upsample network
chm_up = layers.Conv2D(1, 2, activation="relu", padding="same")(chm_input)
chm_up = layers.UpSampling2D(3)(chm_up)
#chm_up = layers.Dropout(0.4)(chm_up)
chm_up = layers.Conv2D(1, 4, activation="relu")(chm_up)
chm_up = layers.Conv2D(1, 4, activation="relu")(chm_up)
chm_up = layers.UpSampling2D(2)(chm_up)
#chm_up = layers.Dropout(0.4)(chm_up)
chm_up = layers.Conv2D(1, 5, activation="relu")(chm_up)
chm_up = layers.Conv2D(1, 5, activation="relu")(chm_up)
chm_up = layers.UpSampling2D(2, name="chm_up")(chm_up)
#chm_up = layers.Dropout(0.4)(chm_up)
# High-res network
high_res = layers.Concatenate(axis=3)([rgb_input, hsi_up, chm_up])
high_res = layers.Conv2D(10, 5, activation="relu")(high_res)
high_res = layers.Conv2D(10, 5, activation="relu")(high_res)
high_res = layers.Conv2D(5, 5, activation="relu", name="high_res")(high_res)
high_res = layers.Flatten()(high_res)
"""
# Low-res network
low_res = layers.Concatenate(axis=3)([hsi_input, chm_input])
low_res = layers.Conv2D(4, 2, activation="relu", padding="same")(low_res)
low_res = layers.Conv2D(8, 2, activation="relu", padding="same")(low_res)
low_res = layers.Conv2D(8, 2, activation="relu", padding="same")(low_res)
low_res = layers.MaxPool2D(2)(low_res)
low_res = layers.Conv2D(16, 2, activation="relu", padding="same")(low_res)
low_res = layers.Conv2D(16, 2, activation="relu", padding="same")(low_res)
low_res = layers.MaxPool2D(2)(low_res)
low_res = layers.Conv2D(32, 2, activation="relu", padding="same")(low_res)
low_res = layers.Conv2D(32, 2, activation="relu", padding="same")(low_res)
low_res = layers.MaxPool2D(2)(low_res)
low_res = layers.Conv2D(64, 2, activation="relu", padding="same")(low_res)
low_res = layers.Conv2D(64, 2, activation="relu", padding="same")(low_res)
low_res = layers.Flatten()(low_res)
# Las 3D network
las_net = layers.Conv3D(2, 4, activation="relu", padding="same")(las_input)
las_net = layers.Conv3D(2, 4, activation="relu", padding="same")(las_net)
las_net = layers.MaxPool3D(2)(las_net)
las_net = layers.Conv3D(8, 4, activation="relu", padding="same")(las_net)
las_net = layers.Conv3D(8, 4, activation="relu", padding="same")(las_net)
las_net = layers.MaxPool3D(2)(las_net)
las_net = layers.Conv3D(16, 4, activation="relu", padding="same")(las_net)
las_net = layers.Conv3D(16, 4, activation="relu", padding="same")(las_net)
las_net = layers.MaxPool3D(2)(las_net)
las_net = layers.Conv3D(32, 4, activation="relu", padding="same")(las_net)
las_net = layers.Conv3D(32, 4, activation="relu", padding="same", name="las_net")(las_net)
las_net = layers.Flatten()(las_net)
# Combine networks with fully connected layers
fully_con = layers.concatenate([low_res, las_net, rgb_down])
fully_con = layers.Dropout(0.1)(fully_con)
fully_con = layers.Dense(256)(fully_con)
fully_con = layers.Dropout(0.4)(fully_con)
fully_con = layers.Dense(256)(fully_con)
fully_con = layers.Dropout(0.4)(fully_con)
fully_con = layers.Dense(256)(fully_con)
fully_con = layers.Dropout(0.4)(fully_con)
fully_con = layers.Dense(256)(fully_con)
fully_con = layers.Dropout(0.0)(fully_con)
output_bounding = layers.Dense(120, kernel_regularizer=keras.regularizers.l2(0.0001))(fully_con)
output_bounding = layers.Reshape((30, 4), name="bounds")(output_bounding)
output_class = layers.Dense(30, activation="sigmoid", kernel_regularizer=keras.regularizers.l2(0.0001), name="labels")(fully_con)
self.model = Model(
inputs=[rgb_input, chm_input, hsi_input, las_input],
outputs=[output_bounding, output_class],
)
self.model.summary()