def test_merge_dot(): (x_train, y_train), (x_test, y_test) = _get_test_data() left = Sequential() left.add(Dense(num_hidden, input_shape=(input_dim, ))) left.add(Activation('relu')) right = Sequential() right.add(Dense(num_hidden, input_shape=(input_dim, ))) right.add(Activation('relu')) model = Sequential() model.add(Merge([left, right], mode='dot', dot_axes=1)) model.add(Dense(num_classes)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') left = Sequential() left.add(Dense(num_hidden, input_shape=(input_dim, ))) left.add(Activation('relu')) right = Sequential() right.add(Dense(num_hidden, input_shape=(input_dim, ))) right.add(Activation('relu')) model = Sequential() model.add(Merge([left, right], mode='dot', dot_axes=[1, 1])) model.add(Dense(num_classes)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
def test_merge_sum(): (x_train, y_train), (x_test, y_test) = _get_test_data() left = Sequential() left.add(Dense(num_hidden, input_shape=(input_dim,))) left.add(Activation('relu')) right = Sequential() right.add(Dense(num_hidden, input_shape=(input_dim,))) right.add(Activation('relu')) model = Sequential() model.add(Merge([left, right], mode='sum')) model.add(Dense(num_class)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, validation_data=([x_test, x_test], y_test)) model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, validation_split=0.1) model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0) model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, shuffle=False) loss = model.evaluate([x_test, x_test], y_test, verbose=0) model.predict([x_test, x_test], verbose=0) model.predict_classes([x_test, x_test], verbose=0) model.predict_proba([x_test, x_test], verbose=0) # test weight saving fname = 'test_merge_sum_temp.h5' model.save_weights(fname, overwrite=True) left = Sequential() left.add(Dense(num_hidden, input_shape=(input_dim,))) left.add(Activation('relu')) right = Sequential() right.add(Dense(num_hidden, input_shape=(input_dim,))) right.add(Activation('relu')) model = Sequential() model.add(Merge([left, right], mode='sum')) model.add(Dense(num_class)) model.add(Activation('softmax')) model.load_weights(fname) os.remove(fname) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') nloss = model.evaluate([x_test, x_test], y_test, verbose=0) assert(loss == nloss) # test serialization config = model.get_config() Sequential.from_config(config) model.summary() json_str = model.to_json() model_from_json(json_str) yaml_str = model.to_yaml() model_from_yaml(yaml_str)
def _align_to_rank_model(self, l_inputs, l_models): l_aligned_models = [ model(input) for model, input in zip(l_models, l_inputs) ] ranker_model = Merge(mode='concat', name='rank_merge')(l_aligned_models[:2]) att_model = Merge(mode='concat', name='att_merge')(l_aligned_models[2:]) att_ranker = Merge( mode='dot', name='att_rank_dot_merge')([ranker_model, att_model]) att_ranker = Model(input=l_inputs, output=att_ranker) return att_ranker
def _align_to_rank_model(self, l_inputs, l_models): l_aligned_models = [ model(input) for model, input in zip(l_models, l_inputs) ] ranker_model = Merge(mode='concat', name='rank_merge')(l_aligned_models[:2]) ranker_model = Masking()(ranker_model) att_model = Merge(mode='concat', name='att_merge')(l_aligned_models[2:]) att_model = Masking()(att_model) att_model = Activation('softmax', name='softmax_attention')(att_model) att_ranker = Merge(mode='dot', dot_axes=-1, name='att_rank_dot_merge')( [ranker_model, att_model]) att_ranker = Model(input=l_inputs, output=att_ranker) return att_ranker
def test_merge_concat(in_tmpdir): (x_train, y_train), (x_test, y_test) = _get_test_data() left = Sequential(name='branch_1') left.add(Dense(num_hidden, input_shape=(input_dim, ), name='dense_1')) left.add(Activation('relu', name='relu_1')) right = Sequential(name='branch_2') right.add(Dense(num_hidden, input_shape=(input_dim, ), name='dense_2')) right.add(Activation('relu', name='relu_2')) model = Sequential(name='merged_branches') model.add(Merge([left, right], mode='concat', name='merge')) model.add(Dense(num_classes, name='final_dense')) model.add(Activation('softmax', name='softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, validation_data=([x_test, x_test], y_test)) model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, validation_split=0.1) model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0) model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0, shuffle=False) loss = model.evaluate([x_test, x_test], y_test, verbose=0) model.predict([x_test, x_test], verbose=0) model.predict_classes([x_test, x_test], verbose=0) model.predict_proba([x_test, x_test], verbose=0) model.get_config() fname = 'test_merge_concat_temp.h5' model.save_weights(fname, overwrite=True) model.fit([x_train, x_train], y_train, batch_size=batch_size, epochs=epochs, verbose=0) model.load_weights(fname) os.remove(fname) nloss = model.evaluate([x_test, x_test], y_test, verbose=0) assert (loss == nloss)
def merge_models(models, nb_classes): for i, model in enumerate(models): model.get_layer(name='flatten_1').name = 'flatten_1_' + str(i) model.get_layer(name='dense_1').name = 'dense_1_' + str(i) model.get_layer(name='dense_2').name = 'dense_2_' + str(i) model.get_layer(name='dropout_1').name = 'dropout_1_' + str(i) for layer in model.layers: layer.trainable = False model.summary() model = Sequential() #model.add(Input(shape=[(),()])) model.add(Merge(models, mode='concat')) model.add( Dense(nb_classes, activation='softmax', trainable=True, name='dense_merge_1')) model.summary() return model
def _align_to_rank_model(self, l_inputs, l_models): l_aligned_models = [ model(this_input) for model, this_input in zip(l_models, l_inputs) ] if len(l_aligned_models) > 1: ranker_model = Merge(mode='sum', name='rank_merge')(l_aligned_models) else: ranker_model = l_aligned_models[0] # ranker_model = Lambda(lambda x: K.mean(x, axis=None), # output_shape=(1,) # )(ranker_model) # ranker_model = Dense(output_dim=1)(ranker_model) att_ranker = Model(input=l_inputs, output=ranker_model) return att_ranker
def merge_models(self, models): for i, model in enumerate(models): model.get_layer(name='flatten_1').name = 'flatten_1_' + str(i) model.get_layer(name='dense_1').name = 'dense_1_' + str(i) model.get_layer(name='dense_2').name = 'dense_2_' + str(i) model.get_layer(name='dropout_1').name = 'dropout_1_' + str(i) for layer in model.layers: layer.trainable = False model.summary() model = Sequential() model.add(Merge(models, mode='concat')) return model
def construct_model_via_translation(self, l_field_translation, l_aux_field_translation, ltr_input, aux_ltr_input, q_att_input, l_field_att_input, l_aux_field_att_input): ranker = self._init_translation_ranker(l_field_translation, ltr_input, q_att_input, l_field_att_input) aux_ranker = self._init_translation_ranker(l_aux_field_translation, aux_ltr_input, q_att_input, l_aux_field_att_input, True) trainer = Sequential() trainer.add( Merge([ranker, aux_ranker], mode=lambda x: x[0] - x[1], output_shape=(1, ), name='training_pairwise')) return ranker, trainer
def test_merge_overlap(in_tmpdir): (x_train, y_train), (x_test, y_test) = _get_test_data() left = Sequential() left.add(Dense(num_hidden, input_shape=(input_dim,))) left.add(Activation('relu')) model = Sequential() model.add(Merge([left, left], mode='sum')) model.add(Dense(num_classes)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2, validation_split=0.1) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, shuffle=False) model.train_on_batch(x_train[:32], y_train[:32]) loss = model.evaluate(x_test, y_test, verbose=0) model.predict(x_test, verbose=0) model.predict_classes(x_test, verbose=0) model.predict_proba(x_test, verbose=0) fname = 'test_merge_overlap_temp.h5' print(model.layers) model.save_weights(fname, overwrite=True) print(model.trainable_weights) model.load_weights(fname) os.remove(fname) nloss = model.evaluate(x_test, y_test, verbose=0) assert(loss == nloss) # test serialization config = model.get_config() Sequential.from_config(config) model.summary() json_str = model.to_json() model_from_json(json_str) yaml_str = model.to_yaml() model_from_yaml(yaml_str)
def make_monster_model(embedding_matrix, word_index, train_embeddings=False, weights_path=None): # # # ================== 1 ======================= # model1 = Sequential() # model1.add(Embedding(len(word_index) + 1, # EMBEDDING_SIZE, # weights=[embedding_matrix], # input_length=MAX_LEN, # trainable=train_embeddings)) # model1.add(TimeDistributed(Dense(OUTPUT_SIZE, activation='relu'))) # model1.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(OUTPUT_SIZE,))) # # # ================== 2 ======================= # model2 = Sequential() # model2.add(Embedding(len(word_index) + 1, # EMBEDDING_SIZE, # weights=[embedding_matrix], # input_length=MAX_LEN, # trainable=train_embeddings)) # model2.add(TimeDistributed(Dense(OUTPUT_SIZE, activation='relu'))) # model2.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(OUTPUT_SIZE,))) # ================== 3 ======================= model3 = Sequential() model3.add( Embedding(len(word_index) + 1, EMBEDDING_SIZE, weights=[embedding_matrix], input_length=MAX_LEN, trainable=train_embeddings)) model3.add( Convolution1D(nb_filter=NUM_FILTERS, filter_length=FILTER_LEN, border_mode='valid', activation='relu', subsample_length=1)) model3.add(Dropout(DROPOUT_PROB)) model3.add( Convolution1D(nb_filter=NUM_FILTERS, filter_length=FILTER_LEN, border_mode='valid', activation='relu', subsample_length=1)) model3.add(GlobalMaxPooling1D()) model3.add(Dropout(DROPOUT_PROB)) model3.add(Dense(OUTPUT_SIZE)) model3.add(Dropout(DROPOUT_PROB)) model3.add(BatchNormalization()) # ================== 4 ======================= model4 = Sequential() model4.add( Embedding(len(word_index) + 1, EMBEDDING_SIZE, weights=[embedding_matrix], input_length=MAX_LEN, trainable=train_embeddings)) model4.add( Convolution1D(nb_filter=NUM_FILTERS, filter_length=FILTER_LEN, border_mode='valid', activation='relu', subsample_length=1)) model4.add(Dropout(DROPOUT_PROB)) model4.add( Convolution1D(nb_filter=NUM_FILTERS, filter_length=FILTER_LEN, border_mode='valid', activation='relu', subsample_length=1)) model4.add(GlobalMaxPooling1D()) model4.add(Dropout(DROPOUT_PROB)) model4.add(Dense(OUTPUT_SIZE)) model4.add(Dropout(DROPOUT_PROB)) model4.add(BatchNormalization()) # ================== 5 ======================= model5 = Sequential() model5.add( Embedding(len(word_index) + 1, EMBEDDING_SIZE, input_length=MAX_LEN, dropout=DROPOUT_PROB, trainable=train_embeddings)) model5.add( LSTM(EMBEDDING_SIZE, dropout_W=DROPOUT_PROB, dropout_U=DROPOUT_PROB)) # ================== 6 ======================= model6 = Sequential() model6.add( Embedding(len(word_index) + 1, EMBEDDING_SIZE, input_length=MAX_LEN, dropout=DROPOUT_PROB, trainable=train_embeddings)) model6.add( LSTM(EMBEDDING_SIZE, dropout_W=DROPOUT_PROB, dropout_U=DROPOUT_PROB)) # ================== Merge all ======================= models_to_merge = [model3, model4, model5, model6] merged_model = Sequential() merged_model.add(Merge(models_to_merge, mode='concat')) merged_model.add(BatchNormalization()) merged_model.add(Dense(OUTPUT_SIZE)) merged_model.add(PReLU()) # merged_model.add(Dropout(DROPOUT_PROB)) # merged_model.add(BatchNormalization()) # merged_model.add(Dense(OUTPUT_SIZE)) # merged_model.add(PReLU()) # merged_model.add(Dropout(DROPOUT_PROB)) # merged_model.add(BatchNormalization()) # merged_model.add(Dense(OUTPUT_SIZE)) # merged_model.add(PReLU()) # merged_model.add(Dropout(DROPOUT_PROB)) # merged_model.add(BatchNormalization()) # merged_model.add(Dense(OUTPUT_SIZE)) # merged_model.add(PReLU()) # merged_model.add(Dropout(DROPOUT_PROB)) # merged_model.add(BatchNormalization()) # merged_model.add(Dense(OUTPUT_SIZE)) # merged_model.add(PReLU()) merged_model.add(Dropout(DROPOUT_PROB)) merged_model.add(BatchNormalization()) merged_model.add(Dense(1)) merged_model.add(Activation('sigmoid')) if weights_path is not None: print 'Restoring weights ... ' merged_model.load_weights(weights_path) merged_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) merged_model.summary() return merged_model