def test_merge_dot():
(x_train, y_train), (x_test, y_test) = _get_test_data()
left = Sequential()
left.add(Dense(num_hidden, input_shape=(input_dim, )))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(num_hidden, input_shape=(input_dim, )))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='dot', dot_axes=1))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
left = Sequential()
left.add(Dense(num_hidden, input_shape=(input_dim, )))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(num_hidden, input_shape=(input_dim, )))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='dot', dot_axes=[1, 1]))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
def test_merge_sum():
(x_train, y_train), (x_test, y_test) = _get_test_data()
left = Sequential()
left.add(Dense(num_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(num_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='sum'))
model.add(Dense(num_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, validation_data=([x_test, x_test], y_test))
model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, validation_split=0.1)
model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0)
model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, shuffle=False)
loss = model.evaluate([x_test, x_test], y_test, verbose=0)
model.predict([x_test, x_test], verbose=0)
model.predict_classes([x_test, x_test], verbose=0)
model.predict_proba([x_test, x_test], verbose=0)
# test weight saving
fname = 'test_merge_sum_temp.h5'
model.save_weights(fname, overwrite=True)
left = Sequential()
left.add(Dense(num_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(num_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='sum'))
model.add(Dense(num_class))
model.add(Activation('softmax'))
model.load_weights(fname)
os.remove(fname)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
nloss = model.evaluate([x_test, x_test], y_test, verbose=0)
assert(loss == nloss)
# test serialization
config = model.get_config()
Sequential.from_config(config)
model.summary()
json_str = model.to_json()
model_from_json(json_str)
yaml_str = model.to_yaml()
model_from_yaml(yaml_str)
def _align_to_rank_model(self, l_inputs, l_models):
l_aligned_models = [
model(input) for model, input in zip(l_models, l_inputs)
]
ranker_model = Merge(mode='concat',
name='rank_merge')(l_aligned_models[:2])
att_model = Merge(mode='concat',
name='att_merge')(l_aligned_models[2:])
att_ranker = Merge(
mode='dot', name='att_rank_dot_merge')([ranker_model, att_model])
att_ranker = Model(input=l_inputs, output=att_ranker)
return att_ranker
def _align_to_rank_model(self, l_inputs, l_models):
l_aligned_models = [
model(input) for model, input in zip(l_models, l_inputs)
]
ranker_model = Merge(mode='concat',
name='rank_merge')(l_aligned_models[:2])
ranker_model = Masking()(ranker_model)
att_model = Merge(mode='concat',
name='att_merge')(l_aligned_models[2:])
att_model = Masking()(att_model)
att_model = Activation('softmax', name='softmax_attention')(att_model)
att_ranker = Merge(mode='dot', dot_axes=-1, name='att_rank_dot_merge')(
[ranker_model, att_model])
att_ranker = Model(input=l_inputs, output=att_ranker)
return att_ranker
def test_merge_concat(in_tmpdir):
(x_train, y_train), (x_test, y_test) = _get_test_data()
left = Sequential(name='branch_1')
left.add(Dense(num_hidden, input_shape=(input_dim, ), name='dense_1'))
left.add(Activation('relu', name='relu_1'))
right = Sequential(name='branch_2')
right.add(Dense(num_hidden, input_shape=(input_dim, ), name='dense_2'))
right.add(Activation('relu', name='relu_2'))
model = Sequential(name='merged_branches')
model.add(Merge([left, right], mode='concat', name='merge'))
model.add(Dense(num_classes, name='final_dense'))
model.add(Activation('softmax', name='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit([x_train, x_train],
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
validation_data=([x_test, x_test], y_test))
model.fit([x_train, x_train],
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
validation_split=0.1)
model.fit([x_train, x_train],
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0)
model.fit([x_train, x_train],
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
shuffle=False)
loss = model.evaluate([x_test, x_test], y_test, verbose=0)
model.predict([x_test, x_test], verbose=0)
model.predict_classes([x_test, x_test], verbose=0)
model.predict_proba([x_test, x_test], verbose=0)
model.get_config()
fname = 'test_merge_concat_temp.h5'
model.save_weights(fname, overwrite=True)
model.fit([x_train, x_train],
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0)
model.load_weights(fname)
os.remove(fname)
nloss = model.evaluate([x_test, x_test], y_test, verbose=0)
assert (loss == nloss)
def merge_models(models, nb_classes):
for i, model in enumerate(models):
model.get_layer(name='flatten_1').name = 'flatten_1_' + str(i)
model.get_layer(name='dense_1').name = 'dense_1_' + str(i)
model.get_layer(name='dense_2').name = 'dense_2_' + str(i)
model.get_layer(name='dropout_1').name = 'dropout_1_' + str(i)
for layer in model.layers:
layer.trainable = False
model.summary()
model = Sequential()
#model.add(Input(shape=[(),()]))
model.add(Merge(models, mode='concat'))
model.add(
Dense(nb_classes,
activation='softmax',
trainable=True,
name='dense_merge_1'))
model.summary()
return model
def _align_to_rank_model(self, l_inputs, l_models):
l_aligned_models = [
model(this_input) for model, this_input in zip(l_models, l_inputs)
]
if len(l_aligned_models) > 1:
ranker_model = Merge(mode='sum',
name='rank_merge')(l_aligned_models)
else:
ranker_model = l_aligned_models[0]
# ranker_model = Lambda(lambda x: K.mean(x, axis=None),
# output_shape=(1,)
# )(ranker_model)
# ranker_model = Dense(output_dim=1)(ranker_model)
att_ranker = Model(input=l_inputs, output=ranker_model)
return att_ranker
def merge_models(self, models):
for i, model in enumerate(models):
model.get_layer(name='flatten_1').name = 'flatten_1_' + str(i)
model.get_layer(name='dense_1').name = 'dense_1_' + str(i)
model.get_layer(name='dense_2').name = 'dense_2_' + str(i)
model.get_layer(name='dropout_1').name = 'dropout_1_' + str(i)
for layer in model.layers:
layer.trainable = False
model.summary()
model = Sequential()
model.add(Merge(models, mode='concat'))
return model
def construct_model_via_translation(self, l_field_translation,
l_aux_field_translation, ltr_input,
aux_ltr_input, q_att_input,
l_field_att_input,
l_aux_field_att_input):
ranker = self._init_translation_ranker(l_field_translation, ltr_input,
q_att_input, l_field_att_input)
aux_ranker = self._init_translation_ranker(l_aux_field_translation,
aux_ltr_input, q_att_input,
l_aux_field_att_input, True)
trainer = Sequential()
trainer.add(
Merge([ranker, aux_ranker],
mode=lambda x: x[0] - x[1],
output_shape=(1, ),
name='training_pairwise'))
return ranker, trainer
def test_merge_overlap(in_tmpdir):
(x_train, y_train), (x_test, y_test) = _get_test_data()
left = Sequential()
left.add(Dense(num_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, left], mode='sum'))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test))
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2, validation_split=0.1)
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0)
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, shuffle=False)
model.train_on_batch(x_train[:32], y_train[:32])
loss = model.evaluate(x_test, y_test, verbose=0)
model.predict(x_test, verbose=0)
model.predict_classes(x_test, verbose=0)
model.predict_proba(x_test, verbose=0)
fname = 'test_merge_overlap_temp.h5'
print(model.layers)
model.save_weights(fname, overwrite=True)
print(model.trainable_weights)
model.load_weights(fname)
os.remove(fname)
nloss = model.evaluate(x_test, y_test, verbose=0)
assert(loss == nloss)
# test serialization
config = model.get_config()
Sequential.from_config(config)
model.summary()
json_str = model.to_json()
model_from_json(json_str)
yaml_str = model.to_yaml()
model_from_yaml(yaml_str)
def make_monster_model(embedding_matrix,
word_index,
train_embeddings=False,
weights_path=None):
# # # ================== 1 =======================
# model1 = Sequential()
# model1.add(Embedding(len(word_index) + 1,
# EMBEDDING_SIZE,
# weights=[embedding_matrix],
# input_length=MAX_LEN,
# trainable=train_embeddings))
# model1.add(TimeDistributed(Dense(OUTPUT_SIZE, activation='relu')))
# model1.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(OUTPUT_SIZE,)))
#
# # ================== 2 =======================
# model2 = Sequential()
# model2.add(Embedding(len(word_index) + 1,
# EMBEDDING_SIZE,
# weights=[embedding_matrix],
# input_length=MAX_LEN,
# trainable=train_embeddings))
# model2.add(TimeDistributed(Dense(OUTPUT_SIZE, activation='relu')))
# model2.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(OUTPUT_SIZE,)))
# ================== 3 =======================
model3 = Sequential()
model3.add(
Embedding(len(word_index) + 1,
EMBEDDING_SIZE,
weights=[embedding_matrix],
input_length=MAX_LEN,
trainable=train_embeddings))
model3.add(
Convolution1D(nb_filter=NUM_FILTERS,
filter_length=FILTER_LEN,
border_mode='valid',
activation='relu',
subsample_length=1))
model3.add(Dropout(DROPOUT_PROB))
model3.add(
Convolution1D(nb_filter=NUM_FILTERS,
filter_length=FILTER_LEN,
border_mode='valid',
activation='relu',
subsample_length=1))
model3.add(GlobalMaxPooling1D())
model3.add(Dropout(DROPOUT_PROB))
model3.add(Dense(OUTPUT_SIZE))
model3.add(Dropout(DROPOUT_PROB))
model3.add(BatchNormalization())
# ================== 4 =======================
model4 = Sequential()
model4.add(
Embedding(len(word_index) + 1,
EMBEDDING_SIZE,
weights=[embedding_matrix],
input_length=MAX_LEN,
trainable=train_embeddings))
model4.add(
Convolution1D(nb_filter=NUM_FILTERS,
filter_length=FILTER_LEN,
border_mode='valid',
activation='relu',
subsample_length=1))
model4.add(Dropout(DROPOUT_PROB))
model4.add(
Convolution1D(nb_filter=NUM_FILTERS,
filter_length=FILTER_LEN,
border_mode='valid',
activation='relu',
subsample_length=1))
model4.add(GlobalMaxPooling1D())
model4.add(Dropout(DROPOUT_PROB))
model4.add(Dense(OUTPUT_SIZE))
model4.add(Dropout(DROPOUT_PROB))
model4.add(BatchNormalization())
# ================== 5 =======================
model5 = Sequential()
model5.add(
Embedding(len(word_index) + 1,
EMBEDDING_SIZE,
input_length=MAX_LEN,
dropout=DROPOUT_PROB,
trainable=train_embeddings))
model5.add(
LSTM(EMBEDDING_SIZE, dropout_W=DROPOUT_PROB, dropout_U=DROPOUT_PROB))
# ================== 6 =======================
model6 = Sequential()
model6.add(
Embedding(len(word_index) + 1,
EMBEDDING_SIZE,
input_length=MAX_LEN,
dropout=DROPOUT_PROB,
trainable=train_embeddings))
model6.add(
LSTM(EMBEDDING_SIZE, dropout_W=DROPOUT_PROB, dropout_U=DROPOUT_PROB))
# ================== Merge all =======================
models_to_merge = [model3, model4, model5, model6]
merged_model = Sequential()
merged_model.add(Merge(models_to_merge, mode='concat'))
merged_model.add(BatchNormalization())
merged_model.add(Dense(OUTPUT_SIZE))
merged_model.add(PReLU())
# merged_model.add(Dropout(DROPOUT_PROB))
# merged_model.add(BatchNormalization())
# merged_model.add(Dense(OUTPUT_SIZE))
# merged_model.add(PReLU())
# merged_model.add(Dropout(DROPOUT_PROB))
# merged_model.add(BatchNormalization())
# merged_model.add(Dense(OUTPUT_SIZE))
# merged_model.add(PReLU())
# merged_model.add(Dropout(DROPOUT_PROB))
# merged_model.add(BatchNormalization())
# merged_model.add(Dense(OUTPUT_SIZE))
# merged_model.add(PReLU())
# merged_model.add(Dropout(DROPOUT_PROB))
# merged_model.add(BatchNormalization())
# merged_model.add(Dense(OUTPUT_SIZE))
# merged_model.add(PReLU())
merged_model.add(Dropout(DROPOUT_PROB))
merged_model.add(BatchNormalization())
merged_model.add(Dense(1))
merged_model.add(Activation('sigmoid'))
if weights_path is not None:
print 'Restoring weights ... '
merged_model.load_weights(weights_path)
merged_model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
merged_model.summary()
return merged_model
