def discriminator_model():
# PatchGAN
inputs = Input(shape=patch_shape)
x = Convolution2D(filters=channel_rate,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(inputs)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Convolution2D(filters=2 * channel_rate,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Convolution2D(filters=4 * channel_rate,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Convolution2D(filters=4 * channel_rate,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = non_local_block(x, mode='embedded', compression=2)
x = Flatten()(x)
outputs = Dense(units=1, activation='sigmoid')(x)
model = Model(inputs=inputs, outputs=outputs, name='PatchGAN')
# model.summary()
# discriminator
inputs = Input(shape=image_shape)
list_row_idx = [(i * channel_rate, (i + 1) * channel_rate)
for i in range(int(image_shape[0] / patch_shape[0]))]
list_col_idx = [(i * channel_rate, (i + 1) * channel_rate)
for i in range(int(image_shape[1] / patch_shape[1]))]
list_patch = []
for row_idx in list_row_idx:
for col_idx in list_col_idx:
x_patch = Lambda(lambda z: z[:, row_idx[0]:row_idx[1], col_idx[0]:
col_idx[1], :])(inputs)
list_patch.append(x_patch)
x = [model(patch) for patch in list_patch]
outputs = Average()(x)
model = Model(inputs=inputs, outputs=outputs, name='Discriminator')
#model.summary()
return model
def Encoder(enc_input, num_feedback_bits):
num_quan_bits = 4
x = tf.transpose(enc_input, perm=[0, 3, 1, 2])
x = tf.keras.layers.Conv2D(2, (3, 3),
padding='same',
data_format="channels_first")(x)
x = add_common_layers(x)
x = non_local_block(x, compression=compression_rate, mode=non_local_mode)
x = tf.keras.layers.Reshape((img_total, ))(x)
x = layers.Dense(units=int(num_feedback_bits / num_quan_bits),
activation='sigmoid')(x)
enc_output = QuantizationLayer(num_quan_bits)(x)
return enc_output
def NON_LOCAL_MIXTURE():
'''NONLOCAL MODEL'''
'''LIDAR branch'''
input_lid = Input(shape=(20, 200, 1))
layer = Conv2D(5,
kernel_size=(5, 5),
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(input_lid)
layer = Conv2D(5,
kernel_size=(5, 5),
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(layer)
layer = Conv2D(5,
kernel_size=(5, 5),
strides=2,
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(layer)
layer = Conv2D(5,
kernel_size=(5, 5),
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(layer)
layer = Conv2D(5,
kernel_size=(5, 5),
strides=2,
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(layer)
NLA = non_local_block(layer, intermediate_dim=2, mode='embedded')
layer = Conv2D(1,
kernel_size=(3, 3),
strides=(1, 2),
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(NLA)
layer = Flatten()(layer)
out_lid = Dense(16, activation='relu')(layer)
'''GPS branch'''
input_coord = Input(shape=(2))
'''Concatenation'''
concatenated = concatenate([out_lid, input_coord])
layer = Dense(64, activation='relu')(concatenated)
layer = Dense(64, activation='relu')(layer)
layer = Dense(64, activation='relu')(layer)
predictions = Dense(256, activation='softmax')(layer)
architecture = Model(inputs=[input_lid, input_coord], outputs=predictions)
return architecture
def build_model():
"""Function returning keras model instance.
Model can be
- Trained here
- Loaded with load_model
- Loaded from keras.applications
"""
input_tensor = Input(shape=(224, 224, 3))
# backbone
base_model = VGG16(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
base_output = base_model.output
# self-attention
x = non_local.non_local_block(base_output,
intermediate_dim=None,
compression=2,
mode='embedded',
add_residual=False)
x = BatchNormalization()(x)
# channel-attention
y = channel_attention.squeeze_excitation_layer(base_output,
512,
ratio=4,
concate=False)
y = BatchNormalization()(y)
# concat
x = concatenate([base_output, x], axis=3)
x = concatenate([x, y], axis=3)
# spp
gap = GlobalAveragePooling2D()(x)
x = Flatten()(x)
x = concatenate([gap, x])
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
predict = Dense(4, activation='softmax')(x)
model = Model(inputs=input_tensor, outputs=predict)
model.load_weights('model.h5')
model.summary()
return model
def f(x):
for i in range(blocks):
x = block_function(filters=filters,
stage=stage,
block=i,
transition_strides=transition_strides[i],
dilation_rate=transition_dilation_rates[i],
is_first_block_of_first_layer=(is_first_layer
and i == 0),
dropout=dropout,
residual_unit=residual_unit)(x)
# Non Local Blook
if filters >= 256:
print("Filters : ", filters, "Adding Non Local Blocks")
x = non_local_block(x, mode='embedded', compression=2)
return x
def Decoder(dec_input, num_feedback_bits):
num_quan_bits = 4
x = DeuantizationLayer(num_quan_bits)(dec_input)
x = tf.keras.layers.Reshape((int(num_feedback_bits / num_quan_bits), ))(x)
x = layers.Dense(img_total, activation='sigmoid')(x)
x = layers.Reshape((2, 24, 16))(x)
x = non_local_block(x, compression=compression_rate, mode=non_local_mode)
for i in range(dense_num):
x = dense_residual_block(x)
x = add_common_layers(x)
x = tf.keras.layers.Conv2D(2, (1, 1),
padding='same',
data_format="channels_first")(x)
x = tf.keras.layers.Conv2D(2, (3, 3),
activation='sigmoid',
padding='same',
data_format="channels_first")(x)
dec_output = tf.transpose(x, perm=[0, 2, 3, 1])
return dec_output
def NonLocal_SeNet_Block(x_input, out_dims, mode=None, compression=2, reduction_ratio=4):
residual_abs = Lambda(abs_backend, name="abs_non" + str(out_dims))(x_input)
x = Conv3D(out_dims, (1, 1, 1), padding='same', use_bias=False,
kernel_initializer='he_normal', kernel_regularizer=l2(5e-4))(x_input)
# NonLocal
x_non_local = non_local_block(x, mode='embedded', compression=2)
# SeNet
abs_mean = GlobalAveragePooling3D()(x_non_local)
# scales = Dense(units=out_dims // reduction_ratio, activation=None, kernel_initializer='he_normal',
# kernel_regularizer=l2(1e-4))(abs_mean)
# scales = Activation('relu')(scales)
# scales = Dense(units=out_dims)(scales)
# scales = Activation('sigmoid')(scales)
# scales = Reshape((1, 1, 1, out_dims))(scales)
scales = Reshape((1, 1, 1, out_dims))(abs_mean)
scales = Conv3D(filters=out_dims // reduction_ratio, kernel_size=1,
use_bias=False, kernel_initializer='he_normal', kernel_regularizer=l2(5e-4))(scales)
scales = Activation('relu')(scales)
scales = Conv3D(filters=out_dims, kernel_size=1,
use_bias=False, kernel_initializer='he_normal', kernel_regularizer=l2(5e-4))(scales)
scales = Activation('sigmoid')(scales)
thres = multiply([x, scales])
# Soft thresholding
sub = keras.layers.subtract([residual_abs, thres])
zeros = keras.layers.subtract([sub, sub])
n_sub = keras.layers.maximum([sub, zeros])
residual = keras.layers.multiply([Lambda(sign_backend, name="sign_non" + str(out_dims))(x_input), n_sub])
return residual
def resnet_MRF():
# create model CNN
"""ResNet Version 1 Model builder
Stacks of BN-ReLU-Conv1D
# Arguments
input_shape (tensor): shape of input image tensor, e.g. 200x1
depth (int): number of core convolutional layers, 6n+2, e.g. 20, 32, 44
num_classes (int): number of classes, e.g.2 types of tissue parameters,
T1 and T2)
# Returns
model (Model): Keras model instance
"""
seq_length = 1000
inputs = Input(shape=(seq_length, 1))
x = Conv1D(16,
21,
strides=1,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-4),
name='block1_conv1')(inputs)
x = Activation('relu')(x)
x = Conv1D(16,
21,
strides=1,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(1e-4),
name='block1_conv2')(x)
# x = BatchNormalization()(x)
x = Activation('relu')(x)
x = non_local_block(
x, compression=1, mode='embedded'
) # mode: `embedded`, `gaussian`, `dot` or `concatenate`.
x = MaxPooling1D(2)(x)
x = resnet_layer(inputs=x,
num_filters=32,
kernel_size=1,
batch_normalization=False,
name='block2_conv1')
y = resnet_layer(inputs=x,
num_filters=32,
kernel_size=21,
batch_normalization=False,
name='block2_conv2')
x = add([x, y])
x = non_local_block(
x, compression=1, mode='embedded'
) # mode: `embedded`, `gaussian`, `dot` or `concatenate`.
x = MaxPooling1D(2)(x)
x = resnet_layer(inputs=x,
num_filters=64,
kernel_size=1,
batch_normalization=False,
name='block3_conv1')
y = resnet_layer(inputs=x,
num_filters=64,
kernel_size=21,
batch_normalization=False,
name='block3_conv2')
x = add([x, y])
x = non_local_block(
x, compression=1, mode='embedded'
) # mode: `embedded`, `gaussian`, `dot` or `concatenate`.
x = MaxPooling1D(2)(x)
x = resnet_layer(inputs=x,
num_filters=128,
kernel_size=1,
batch_normalization=False,
name='block4_conv1')
y = resnet_layer(inputs=x,
num_filters=128,
kernel_size=21,
batch_normalization=False,
name='block4_conv2')
x = add([x, y])
x = non_local_block(
x, compression=2, mode='embedded'
) # mode: `embedded`, `gaussian`, `dot` or `concatenate`.
x = GlobalAveragePooling1D()(x)
outputs = Dense(2,
kernel_initializer='he_normal',
kernel_constraint=maxnorm(3))(x)
# Instantiate model.
model = Model(inputs=inputs, outputs=outputs)
return model
def Network_config(class_num=4,
epoch=200,
initial_epoch=0,
batch_size=32,
train_data=None,
train_label=None,
test_data=None,
test_label=None,
fold=0):
adam = Adam(lr=0.005,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0009)
sgd = SGD(lr=0.001, momentum=0.9, decay=0.0, nesterov=False)
input_tensor = Input(shape=(224, 224, 3))
#backbone
base_model = VGG16(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
base_output = base_model.output
#self-attention
x = non_local.non_local_block(base_output,
intermediate_dim=None,
compression=2,
mode='embedded',
add_residual=False)
x = BatchNormalization()(x)
#channel-attention
y = channel_attention.squeeze_excitation_layer(base_output,
512,
ratio=4,
concate=False)
y = BatchNormalization()(y)
#concat
x = concatenate([base_output, x], axis=3)
x = concatenate([x, y], axis=3)
# spp
gap = GlobalAveragePooling2D()(x)
x = Flatten()(x)
x = concatenate([gap, x])
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
predict = Dense(class_num, activation='softmax')(x)
model = Model(inputs=input_tensor, outputs=predict)
for layer in (base_model.layers):
layer.trainable = False
for l in model.layers:
print(l.name)
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=[keras.metrics.categorical_accuracy])
model.summary()
tools.create_directory('./final/')
weights_file = './final/' + str(
fold
) + '-weights.{epoch:02d}-{categorical_accuracy:.4f}-{val_loss:.4f}-{val_categorical_accuracy:.4f}.h5'
csv_file = './final/record.csv'
lr_reducer = ReduceLROnPlateau(monitor='categorical_accuracy',
factor=0.2,
cooldown=0,
patience=2,
min_lr=0.5e-6)
early_stopper = EarlyStopping(monitor='val_categorical_accuracy',
min_delta=1e-4,
patience=30)
model_checkpoint = ModelCheckpoint(weights_file,
monitor='val_categorical_accuracy',
save_best_only=True,
verbose=1,
save_weights_only=True,
mode='max')
tensorboard = TensorBoard(log_dir='./logs/',
histogram_freq=0,
batch_size=8,
write_graph=True,
write_grads=True,
write_images=True,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
CSV_record = CSVLogger(csv_file, separator=',', append=True)
callbacks = [
lr_reducer, early_stopper, model_checkpoint, tensorboard, CSV_record
]
gc.disable()
model.fit_generator(
generator=tools.batch_generator(np.array(train_data),
np.array(train_label), batch_size,
True, class_num, True),
steps_per_epoch=int(len(train_label) / batch_size) - 1,
max_q_size=20,
initial_epoch=initial_epoch,
epochs=epoch,
verbose=1,
callbacks=callbacks,
validation_data=tools.batch_generator(np.array(test_data),
np.array(test_label), batch_size,
True, class_num, False),
validation_steps=int(len(test_label) / batch_size) - 1,
class_weight='auto')
#confusion matrix
all_y_pred = []
all_y_true = []
for test_data_batch, test_label_batch in tools.batch_generator_confusion_matrix(
np.array(test_data), np.array(test_label), batch_size, True,
class_num):
y_pred = model.predict(test_data_batch, batch_size)
y_true = test_label_batch
for y_p in y_pred:
all_y_pred.append(np.where(y_p == max(y_p))[0][0])
for y_t in y_true:
all_y_true.append(np.where(y_t == max(y_t))[0][0])
confusion = confusion_matrix(y_true=all_y_true, y_pred=all_y_pred)
print(confusion)
f = open('confusion_matrix.txt', 'a+')
f.write(str(all_y_true) + "\n")
f.write(str(all_y_pred) + "\n")
f.write(str(confusion) + '\n')
f.close()
gc.enable()
def get_cnn_model(params):
"""
Load base CNN model and add metadata fusion layers if 'use_metadata' is set in params.py
:param params: global parameters, used to find location of the dataset and json file
:return model: CNN model with or without depending on params
"""
input_tensor = Input(shape=(params.target_img_size[0],
params.target_img_size[1],
params.num_channels))
baseModel = densenet.DenseNetImageNet161(
input_shape=(params.target_img_size[0], params.target_img_size[1],
params.num_channels),
include_top=False,
input_tensor=input_tensor)
modelStruct = baseModel.layers[-1].output
if params.use_nlm:
modelStruct = baseModel.layers[-2].output
modelStruct = non_local_block(modelStruct,
computation_compression=1,
mode='embedded')
modelStruct = Conv2D(params.cnn_lstm_layer_length, [3, 3],
name='conv_nlm')(modelStruct)
modelStruct = Flatten()(modelStruct)
modelStruct = Dense(params.cnn_lstm_layer_length,
activation='relu',
name='fc_nlm')(modelStruct)
modelStruct = Dropout(0.5)(modelStruct)
if params.use_spp:
modelStruct = baseModel.layers[-2].output
modelStruct = SpatialPyramidPooling([1, 2, 4], name='spp')(modelStruct)
modelStruct = Dense(params.cnn_lstm_layer_length,
activation='relu',
name='fc_spp')(modelStruct)
modelStruct = Dropout(0.5)(modelStruct)
if params.use_deform:
modelStruct = baseModel.layers[-2].output
modelStruct = ConvOffset2D(params.cnn_lstm_layer_length,
name='deform')(modelStruct)
modelStruct = Conv2D(params.cnn_lstm_layer_length, [3, 3],
name='conv_deform')(modelStruct)
modelStruct = Flatten()(modelStruct)
modelStruct = Dense(params.cnn_lstm_layer_length,
activation='relu',
name='fc_deform')(modelStruct)
modelStruct = Dropout(0.5)(modelStruct)
if params.use_metadata:
auxiliary_input = Input(shape=(params.metadata_length, ),
name='aux_input')
modelStruct = merge([modelStruct, auxiliary_input], 'concat')
modelStruct = Dense(params.cnn_last_layer_length,
activation='relu',
name='fc1')(modelStruct)
modelStruct = Dropout(0.5)(modelStruct)
modelStruct = Dense(params.cnn_last_layer_length,
activation='relu',
name='fc2')(modelStruct)
modelStruct = Dropout(0.5)(modelStruct)
predictions = Dense(params.num_labels, activation='softmax')(modelStruct)
if not params.use_metadata:
model = Model(input=[baseModel.input], output=predictions)
else:
model = Model(input=[baseModel.input, auxiliary_input],
output=predictions)
for i, layer in enumerate(model.layers):
layer.trainable = True
return model
def NON_LOCAL_MIXTURE_OLD(FLATTENED, LIDAR_TYPE):
if (LIDAR_TYPE == 'CENTERED'):
if (FLATTENED):
input_lid = Input(shape=(67, 67, 1))
else:
input_lid = Input(shape=(67, 67, 10))
elif (LIDAR_TYPE == 'ABSOLUTE'):
if (FLATTENED):
input_lid = Input(shape=(20, 200, 1))
else:
input_lid = Input(shape=(20, 200, 10))
elif (LIDAR_TYPE == 'ABSOLUTE_LARGE'):
if (FLATTENED):
input_lid = Input(shape=(60, 330, 1))
else:
input_lid = Input(shape=(60, 330, 10))
noisy_input_lid = GaussianNoise(0.01)(input_lid)
layer = Conv2D(5,
kernel_size=(5, 5),
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(noisy_input_lid)
layer = Conv2D(5,
kernel_size=(5, 5),
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(layer)
layer = Conv2D(5,
kernel_size=(5, 5),
strides=2,
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(layer)
layer = Conv2D(5,
kernel_size=(5, 5),
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(layer)
layer_input = Conv2D(5,
kernel_size=(5, 5),
strides=2,
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(layer)
non_local_layer = non_local_block(layer_input, intermediate_dim=2)
layer = Conv2D(1,
kernel_size=(3, 3),
strides=(1, 2),
activation='relu',
padding="SAME",
kernel_initializer=initializers.HeUniform)(non_local_layer)
layer = Flatten()(layer)
out_lid = Dense(16, activation='relu')(layer)
'''GPS branch'''
input_coord = Input(shape=(3))
noisy_input_coord = GaussianNoise(0.002)(input_coord)
'''Concatenation'''
concatenated = concatenate([out_lid, noisy_input_coord])
reg_val = 0
layer = Dense(64, activation='relu')(concatenated)
layer = Dense(64, activation='relu')(layer)
layer = Dense(64, activation='relu')(layer)
predictions = Dense(256, activation='softmax')(layer)
architecture = Model(inputs=[input_lid, input_coord], outputs=predictions)
return architecture
super(CustomModelCheckpoint, self).__init__()
self.save_model = model_parallel
self.path = path
self.nb_epoch = 0
def on_epoch_end(self, epoch, logs=None):
self.nb_epoch += 1
self.save_model.save(self.path + str(self.nb_epoch) + '.hdf5')
i3d = i3d_modified(weights = 'rgb_imagenet_and_kinetics')
model_branch = i3d.i3d_flattened(num_classes = num_classes)
model_branch.load_weights('/data/stars/user/sdas/PhD_work/ICCV_2019/models/epoch_full_body_NTU_CS.hdf5')
model_i3d = Model(inputs = model_branch.input, outputs = model_branch.get_layer('Mixed_5c').output)
x = non_local_block(model_i3d.output, compression=2, mode='embedded')
#x = non_local_block(x, compression=2, mode='embedded')
#x = non_local_block(x, compression=2, mode='embedded')
#x = non_local_block(x, compression=2, mode='embedded')
#x = non_local_block(x, compression=2, mode='embedded')
x = AveragePooling3D((2, 7, 7), strides=(1, 1, 1), padding='valid', name='global_avg_pool'+'second')(x)
x = Dropout(0.0)(x)
x = conv3d_bn(x, num_classes, 1, 1, 1, padding='same', use_bias=True, use_activation_fn=False, use_bn=False, name='Conv3d_6a_1x1'+'second')
x = Flatten(name='flatten'+'second')(x)
predictions = Dense(num_classes, activation='softmax', name='softmax'+'second')(x)
model = Model(inputs=model_branch.input, outputs=predictions, name = 'i3d_nonlocal')
optim = SGD(lr = 0.01, momentum = 0.9)
model.compile(loss = 'categorical_crossentropy', optimizer = optim, metrics = ['accuracy'])
#model = load_model("../weights3/epoch11.hdf5")
# Callbacks