def build_model(layer_sizes1, layer_sizes2, input_size1, input_size2,
learning_rate, reg_par, outdim_size, use_all_singular_values,
neurons):
input1 = Input(shape=(1, input_size1), name='audios')
input2 = Input(shape=(input_size2, ), name='lyrics')
activation_model = 'sigmoid'
dense1_1 = Dense(neurons,
activation=activation_model,
name='view_1_1',
kernel_regularizer=l2(reg_par))(input1)
dense1_2 = Dense(neurons,
activation=activation_model,
name='view_1_2',
kernel_regularizer=l2(reg_par))(dense1_1)
dense1_3 = Dense(neurons,
activation=activation_model,
name='view_1_3',
kernel_regularizer=l2(reg_par))(dense1_2)
output1 = Dense(outdim_size,
activation='linear',
name='view_1_4',
kernel_regularizer=l2(reg_par))(dense1_3)
dense2_1 = Dense(neurons,
activation=activation_model,
name='view_2_1',
kernel_regularizer=l2(reg_par))(input2)
dense2_2 = Dense(neurons,
activation=activation_model,
name='view_2_2',
kernel_regularizer=l2(reg_par))(dense2_1)
dense2_3 = Dense(neurons,
activation=activation_model,
name='view_2_3',
kernel_regularizer=l2(reg_par))(dense2_2)
output2 = Dense(outdim_size,
activation='linear',
name='view_2_4',
kernel_regularizer=l2(reg_par))(dense2_3)
added = Add()([output1, output2
]) # equivalent to added = keras.layers.add([x1, x2])
out = Dense(20)(added)
model = Model(inputs=[input1, input2], outputs=out)
model_optimizer = Adam(lr=learning_rate)
model.compile(loss=cca_loss(outdim_size, use_all_singular_values),
optimizer=model_optimizer)
return model
def build_DRNN_model(input_size1, input_size2, dense_size, learning_rate,
reg_par, outdim_size, activation_lstm,
use_all_singular_values, neurons_lstm, dropout,
activation_model1, activation_model2, activation_model3,
neurons1, neurons2, neurons3):
input1 = Input(shape=(1, input_size1), name='audios')
input2 = Input(shape=(input_size2, ), name='lyrics')
output1_lstm = create_BLSTM_model(input1, input_size1, dropout, reg_par,
outdim_size, activation_lstm)
batchno_1 = BatchNormalization()(output1_lstm)
dense1_1 = Dense(neurons1,
activation=activation_model1,
name='view_1_1',
kernel_regularizer=l2(reg_par))(output1_lstm)
batchno_2 = BatchNormalization()(dense1_1)
dense1_2 = Dense(neurons2,
activation=activation_model2,
name='view_1_2',
kernel_regularizer=l2(reg_par))(batchno_2)
batchno_3 = BatchNormalization()(dense1_2)
output1 = Dense(20,
activation='linear',
name='view_1_4',
kernel_regularizer=l2(reg_par))(batchno_3)
dense2_1 = Dense(input_size2,
activation=activation_model1,
name='view_2_1',
kernel_regularizer=l2(reg_par))(input2)
dense2_2 = Dense(neurons2,
activation=activation_model2,
name='view_2_2',
kernel_regularizer=l2(reg_par))(dense2_1)
output2 = Dense(20,
activation='linear',
name='view_2_4',
kernel_regularizer=l2(reg_par))(dense2_2)
added = Add()([output1, output2
]) # equivalent to added = keras.layers.add([x1, x2])
out = Dense(20)(added)
model = Model(inputs=[input1, input2], outputs=out)
model_optimizer = RMSprop(lr=learning_rate)
model.compile(loss=cca_loss(outdim_size, False), optimizer=model_optimizer)
return model
def create_model(layer_sizes1, layer_sizes2, input_size1, input_size2,
learning_rate, reg_par, outdim_size, use_all_singular_values):
"""
builds the whole model
the structure of each sub-network is defined in build_mlp_net,
and it can easily get substituted with a more efficient and powerful network like CNN
"""
view1_model = build_mlp_net(layer_sizes1, input_size1, reg_par)
view2_model = build_mlp_net(layer_sizes2, input_size2, reg_par)
model = Sequential()
ipdb.set_trace()
##Merge not working
model.add(Merge([view1_model, view2_model]))
model_optimizer = RMSprop(lr=learning_rate)
model.compile(loss=cca_loss(outdim_size, use_all_singular_values),
optimizer=model_optimizer)
return model
def create_CNN_RNN(height, width, depth, input_size2, dense_size,
learning_rate, reg_par, outdim_size, activation_lstm,
use_all_singular_values, neurons_lstm, dropout,
activation_model1, activation_model2, activation_model3,
neurons1, neurons2, neurons3):
kernel_size_1 = 3 # we will use 3x3 kernels throughout
kernel_size_2 = 3
kernel_size_3 = 2
pool_size_1 = 2
pool_size_2 = 1 # we will use 2x2 pooling throughout
conv_depth_1 = 1 # we will initially have 32 kernels per conv. layer...
conv_depth_2 = 3 # ...switching to 64 after the first pooling layer
conv_depth_3 = 1
drop_prob_1 = 0.25 # dropout after pooling with probability 0.25
drop_prob_2 = 0.5 # dropout in the FC layer with probability 0.5
hidden_size = 20 # the FC layer will have 512 neurons
input1 = Input(
shape=(height, width, depth), name='audios'
) # depth goes last in TensorFlow back-end (first in Theano)
conv_1 = Convolution2D(conv_depth_1, (kernel_size_1, kernel_size_1),
kernel_regularizer=l2(reg_par),
padding='same',
activation='softsign')(input1)
pool_1 = MaxPooling2D(pool_size=(pool_size_1, pool_size_1))(conv_1)
drop_1 = Dropout(drop_prob_2)(pool_1)
conv_2 = Convolution2D(conv_depth_2, (kernel_size_2, kernel_size_2),
kernel_regularizer=l2(reg_par),
padding='same',
activation='softsign')(drop_1)
pool_2 = MaxPooling2D(pool_size=(pool_size_2, pool_size_2))(conv_2)
drop_2 = Dropout(drop_prob_2)(pool_2)
conv_3 = Convolution2D(conv_depth_3, (kernel_size_3, kernel_size_3),
kernel_regularizer=l2(reg_par),
padding='same',
activation='softsign')(drop_2)
pool_3 = MaxPooling2D(pool_size=(pool_size_1, pool_size_2))(conv_3)
# Now flatten to 1D, apply FC -> ReLU (with dropout) -> softmax
drop_3 = Dropout(drop_prob_1)(pool_3)
reshape = Reshape((25, 1), input_shape=(5, 5, 1))(drop_3)
blstm = Bidirectional(
LSTM(25,
stateful=False,
activation="softsign",
return_sequences=True,
kernel_regularizer=l2(reg_par)))(reshape)
blstm_2 = Bidirectional(
LSTM(25,
stateful=False,
activation="softsign",
return_sequences=True,
kernel_regularizer=l2(reg_par)))(blstm)
flat = Flatten()(blstm_2)
output1 = Dense(512,
activation='linear',
kernel_regularizer=l2(reg_par),
name='view_1_3')(flat)
input2 = Input(shape=(input_size2, ), name='lyrics')
batchno_2 = BatchNormalization()(input2)
dense2_1 = Dense(512,
activation='relu',
kernel_regularizer=l2(reg_par),
name='view_2_1')(batchno_2)
dense2_2 = Dense(512,
activation='relu',
kernel_regularizer=l2(reg_par),
name='view_2_2')(dense2_1)
output2 = Dense(512,
activation='linear',
kernel_regularizer=l2(reg_par),
name='view_2_3')(dense2_2)
added = Add()([output1, output2
]) # equivalent to added = keras.layers.add([x1, x2])
out = Dense(1024, activation='linear', name='output')(added)
model = Model(inputs=([input1, input2]), outputs=out)
model_optimizer = RMSprop(lr=0.001)
model_optimizer_1 = Adam(lr=0.001)
model.compile(loss=cca_loss(1024, True),
optimizer=model_optimizer_1,
metrics=["acc", top_accuracy])
print(model.summary())
return model