def parallel_cross_straight_newactivation(): log.write("\nparallel-straight-cross-newactivation\n") conv_0 = Conv1D(64,kernel_size,padding='valid',strides=1, activation=newacti)(e) maxpool_0 = MaxPooling1D(pool_size=12)(conv_0) gru=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(e) maxpool_gru = MaxPooling1D(pool_size=12)(gru) merge2 = maximum([maxpool_0, maxpool_gru]) #merge=Reshape((sequence_length,filters1))(merge2) gru1=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(merge2) gru1=MaxPooling1D(pool_size=8)(gru1) conv_1=Conv1D(filters1,kernel_size,padding='valid',activation=newacti,strides=1)(merge2) maxpool_1=MaxPooling1D(pool_size=8)(conv_1) merge1 = maximum([gru1,maxpool_1]) flatten = Flatten()(merge1) dropout = Dropout(0.5)(flatten) output = Dense(nclass, activation='softmax')(dropout) model = Model(inputs=visible, outputs=output) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['top_k_categorical_accuracy']) print(model.summary()) model.fit(padded_docs,y_train, nb_epoch=epochesno, batch_size=64, validation_data=(vpadded_docs, y_valid), callbacks=[metriczs, plot]) print('Model built successfully...Please wait.....Evaluating......') # Final evaluation of the model scores = model.evaluate(tpadded_docs, y_test) print("Loss: %.2f%%" % (scores[0]*100)) print("Accuracy: %.2f%%" % (scores[1]*100)) log.write('\nparallel-cross-straight-newactivation-accuracy -\n '+str(scores[1]*100)+"\t Loss: "+str(scores[0]*100))
def build_dnn_model(self): layers = self.loader['layers'] maxout = self.loader['maxout'] x = inputs = Input(shape=len(self.columns)) auxiliary_input = Dense(1, activation='relu', kernel_constraint='glorot_uniform')(x) for layer in layers: if layer == 0: continue # x = Dense(layer, activation='sigmoid')(x) if maxout < 2: x = Dense(layer, activation='sigmoid', kernel_initializer='glorot_uniform')(x) else: x = maximum([ Dense(layer, activation='sigmoid', kernel_regularizer=regularizers.l1(0.001), kernel_initializer='one')(x) for _ in range(maxout) ]) x = LeakyReLU(alpha=0.1)(x) x = Dense(1, activation='relu')(x) x = Average()([x, auxiliary_input]) outputs = LeakyReLU(alpha=-1)(x) self.model = Model(inputs=[inputs], outputs=outputs) return self.model
def parallel_cross_newactivation(): embedding = Embedding(top_words,embedding_vecor_length, input_length=max_review_length, trainable=True)(visible) e=Reshape((sequence_length,embedding_vecor_length,1))(embedding) print(e.shape) log.write("\nparallel-cross-newactivation\n") conv_0 = Conv2D(filters1, kernel_size=(filter_sizes[0], 100), padding='valid', kernel_initializer='normal', activation=newacti)(e) maxpool_0 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] +1, 1), strides=(1,1), padding='valid')(conv_0) maxpool_0=Reshape((1,gru_output_size))(maxpool_0) maxpool_0=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(maxpool_0) maxpool_0=Reshape((1,gru_output_size))(maxpool_0) gru=Reshape((sequence_length,embedding_vecor_length))(e) gru=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(gru) gru=Conv1D(64,kernel_size,padding='valid',activation=newacti,strides=1)(gru) gru=MaxPooling1D(pool_size=pool_size)(gru) merge = maximum([maxpool_0, gru]) flatten = Flatten()(merge) dropout = Dropout(0.5)(flatten) output = Dense(nclass, activation='softmax')(dropout) model = Model(inputs=visible, outputs=output) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['top_k_categorical_accuracy']) print(model.summary()) plot_model(model, to_file='model_plot_parallel_cross.png', show_shapes=True, show_layer_names=True) model.fit(X_train, out_train, nb_epoch=epochesno, batch_size=64, validation_data=(X_val,out_val), callbacks=[metriczs]) print('Model built successfully...Please wait.....Evaluating......') # Final evaluation of the model scores = model.evaluate(X_test, out_test, verbose=0) print("Loss: %.2f%%" % (scores[0]*100)) print("Accuracy: %.2f%%" % (scores[1]*100)) log.write('\nparallel-cross-newactivation-accuracy -\n '+str(scores[1]*100)+"\t Loss: "+str(scores[0]*100))
def parallel_cross_leakyrelu(): log.write("\nparallel-cross-leakyrelu\n") conv_0 = Conv1D(64,kernel_size,padding='valid',strides=1, activation=LeakyReLU(alpha=.001))(e) maxpool_0 = MaxPooling1D(pool_size=pool_size)(conv_0) maxpool_0=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(maxpool_0) gru=Reshape((sequence_length,embedding_vecor_length))(e) gru=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(gru) gru=Conv1D(64,kernel_size,padding='valid',activation=LeakyReLU(alpha=.001),strides=1)(gru) gru=MaxPooling1D(pool_size=pool_size)(gru) merge = maximum([maxpool_0, gru]) flatten = Flatten()(merge) dropout = Dropout(0.5)(flatten) output = Dense(nclass, activation='softmax')(dropout) model = Model(inputs=visible, outputs=output) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['top_k_categorical_accuracy']) print(model.summary()) #plot_model(model, to_file='model_plot_parallel_cross.png', show_shapes=True, show_layer_names=True) model.fit(padded_docs,y_train, nb_epoch=epochesno, batch_size=64, validation_data=(vpadded_docs, y_valid), callbacks=[metriczs, plot]) print('Model built successfully...Please wait.....Evaluating......') # Final evaluation of the model scores = model.evaluate(tpadded_docs, y_test) print("Loss: %.2f%%" % (scores[0]*100)) print("Accuracy: %.2f%%" % (scores[1]*100)) log.write('\nparallel-cross-leakyrelu-accuracy -\n '+str(scores[1]*100)+"\t Loss: "+str(scores[0]*100))
def scse_block(inp, ratio=2): x1 = cse_block(inp, ratio) x2 = sse_block(inp) x = maximum([x1, x2]) return x
def create_model(self, X, wv_model): X_preprocessed = super(ModelContainer9, self).inputs_context_emb_layer_nc(X, wv_model) X_preprocessed = np.array([ sequence.pad_sequences(subX_prep, maxlen=maxlen, padding='post', truncating='post', value=-1) for subX_prep in X_preprocessed ]) input = Input(shape=(3, maxlen), dtype='int32') # Left Context branch input_L = Lambda(lambda x: x[:, 0], output_shape=(maxlen, ))(input) embedding_layer_L = wv_model.wv.get_embedding_layer() mask_L = Masking(mask_value=-1)(input_L) emb_seq_L = embedding_layer_L(mask_L) x_L = Conv1D(filters=8, kernel_size=3, activation='relu')(emb_seq_L) x_L = Dropout(0.4)(x_L) x_L = GlobalMaxPooling1D()(x_L) x_L = Dropout(0.2)(x_L) x_L = Dense(2, activation='sigmoid')(x_L) # Connective branch input_NC = Lambda(lambda x: x[:, 1], output_shape=(maxlen, ))(input) embedding_layer_NC = wv_model.wv.get_embedding_layer() mask_NC = Masking(mask_value=-1)(input_NC) emb_seq_NC = embedding_layer_NC(mask_NC) x_NC = Conv1D(filters=128, kernel_size=4, activation='relu')(emb_seq_NC) x_NC = (AveragePooling1D(pool_size=4))(x_NC) x_NC = (Dropout(0.2))(x_NC) x_NC = (Conv1D(filters=256, kernel_size=2, activation='relu'))(x_NC) x_NC = (Dropout(0.5))(x_NC) x_NC = (Conv1D(filters=32, kernel_size=8, activation='relu'))(x_NC) x_NC = GlobalMaxPooling1D()(x_NC) x_NC = Dense(16, activation='tanh')(x_NC) x_NC = Dropout(0.2)(x_NC) x_NC = Dense(2, activation='sigmoid')(x_NC) # Right context branch input_R = Lambda(lambda x: x[:, 2], output_shape=(maxlen, ))(input) embedding_layer_R = wv_model.wv.get_embedding_layer() mask_R = Masking(mask_value=-1)(input_R) emb_seq_R = embedding_layer_R(mask_R) x_R = Conv1D(filters=8, kernel_size=3, activation='relu')(emb_seq_R) x_R = Dropout(0.4)(x_R) x_R = GlobalMaxPooling1D()(x_R) x_R = Dropout(0.2)(x_R) x_R = Dense(2, activation='sigmoid')(x_R) x = maximum([x_L, x_R]) x = concatenate([x_NC, x]) preds = (Dense(2, activation='sigmoid'))(x) model = Model(input, preds, name='Model9_contexts') return model, X_preprocessed
def _build_entity_capsule(self, x, entity, capsules): """Construye una cápsula de entidad particular para la entidad `entity`. Si la entidad es hoja (no tiene hijos) se conecta directamente a `x`, de lo contrario, se conecta a los respectivos hijos. """ children = [capsules[c] for c in entity.children] inputs = maximum(children, name="max-%s" % entity.name) if len(children) > 1 else x return EntityLayer(self.entity_shape, name=entity.name)(inputs)
def eltwise(layer, layer_in, layerId): out = {} if (layer['params']['operation'] == 0): # This input reverse is to handle visualization out[layerId] = multiply(layer_in[::-1]) elif (layer['params']['operation'] == 1): out[layerId] = add(layer_in[::-1]) else: out[layerId] = maximum(layer_in[::-1]) return out
def test_merge_maximum(): i1 = layers.Input(shape=(4, 5)) i2 = layers.Input(shape=(4, 5)) o = layers.maximum([i1, i2]) assert o._keras_shape == (None, 4, 5) model = models.Model([i1, i2], o) x1 = np.random.random((2, 4, 5)) x2 = np.random.random((2, 4, 5)) out = model.predict([x1, x2]) assert out.shape == (2, 4, 5) assert_allclose(out, np.maximum(x1, x2), atol=1e-4)
def build_model(): vocb_size = 0 vocb_size = X_pad.max() + 1 # channel 1 inputs1 = Input(shape=(X_train.shape[1], )) embedding1 = Embedding(vocb_size, embed_size)(inputs1) x_lstm1 = Bidirectional(LSTM(50, activation='relu', return_sequences=True))(embedding1) conv1 = Conv1D(filters=32, kernel_size=4, activation='relu')(x_lstm1) drop1 = Dropout(0.5)(conv1) pool1 = MaxPooling1D(pool_size=2)(drop1) flat1 = Flatten()(pool1) # channel 2 inputs2 = Input(shape=(X_train.shape[1], )) embedding2 = Embedding(vocb_size, 100)(inputs2) conv2 = Conv1D(filters=32, kernel_size=6, activation='relu')(embedding2) drop2 = Dropout(0.5)(conv2) pool2 = MaxPooling1D(pool_size=2)(drop2) flat2 = Flatten()(pool2) merged1 = concatenate([flat1, flat2]) # channel 3 inputs3 = Input(shape=(X_train.shape[1], )) embedding3 = Embedding(vocb_size, embed_size)(inputs3) x_lstm3 = Bidirectional(LSTM(50, activation='relu', return_sequences=True))(embedding3) conv3 = Conv1D(filters=32, kernel_size=4, activation='relu')(x_lstm3) drop3 = Dropout(0.5)(conv3) pool3 = MaxPooling1D(pool_size=2)(drop3) flat3 = Flatten()(pool3) # channel 4 inputs4 = Input(shape=(X_train.shape[1], )) embedding4 = Embedding(vocb_size, 100)(inputs4) conv4 = Conv1D(filters=32, kernel_size=6, activation='relu')(embedding4) drop4 = Dropout(0.5)(conv4) pool4 = MaxPooling1D(pool_size=2)(drop4) flat4 = Flatten()(pool4) merged2 = concatenate([flat3, flat4]) # merge #merged = minimum([merged1,merged2]) merged = maximum([merged1, merged2]) # interpretation dense1 = Dense(10, activation='relu')(merged) outputs = Dense(3, activation='sigmoid')(dense1) model = Model(inputs=[inputs1, inputs2, inputs3, inputs4], outputs=outputs) # compile model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # summarize print(model.summary()) #plot_model(model, show_shapes=True, to_file='multichannel.png') return model
def eltwise(layer, layer_in, layerId): out = {} if (layer['params']['layer_type'] == 'Multiply'): # This input reverse is to handle visualization out[layerId] = multiply(layer_in[::-1]) elif (layer['params']['layer_type'] == 'Sum'): out[layerId] = add(layer_in[::-1]) elif (layer['params']['layer_type'] == 'Average'): out[layerId] = average(layer_in[::-1]) elif (layer['params']['layer_type'] == 'Dot'): out[layerId] = dot(layer_in[::-1], -1) else: out[layerId] = maximum(layer_in[::-1]) return out
def model3(train_x,train_y): """建立四层的神经网络""" inputs=Input(shape=(train_x.shape[1],)) x1 = Dense(500, activation='tanh')(inputs) x2 = Dense(500, activation='relu')(inputs) x=maximum([x1,x2]) x = Dense(50, activation='sigmoid')(x) x = Dense(50, activation='linear')(x) predictions = Dense(10, activation='softmax')(x) model = Model(inputs=inputs, outputs=predictions) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) model.fit(train_x, train_y,epochs=15, batch_size=200, validation_split=0.0) return model
def resnet_mvcnn(target_size, num_images, num_classes): # this is the Network to be share amongst the views img_input = Input(shape=target_size + (3, )) x = Conv2D(64, (7, 7), strides=(2, 2), padding='same', name='conv1')(img_input) # x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) x = Activation('relu')(x) x = MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') x = AveragePooling2D((7, 7), name='avg_pool')(x) outp = Flatten()(x) shared_resnet = Model(img_input, outp) # one input per image inputs = [Input(shape=target_size + (3, )) for _ in range(num_images)] # encode through the shared network encodeds = [shared_resnet(inputs[idx]) for idx in range(num_images)] # rather than concatenate, this time we take the maximum and pass it through another network maximum_tensor = maximum(encodeds) predictions = Dense(num_classes, activation='softmax', name='final_fc')(maximum_tensor) return Model(inputs=inputs, outputs=predictions)
def channel_spatial_squeeze_excite(input, ratio=16): ''' Create a spatial squeeze-excite block Args: input: input tensor filters: number of output filters Returns: a keras tensor References - [Squeeze and Excitation Networks](https://arxiv.org/abs/1709.01507) - [Concurrent Spatial and Channel Squeeze & Excitation in Fully Convolutional Networks](https://arxiv.org/abs/1803.02579) ''' cse = squeeze_excite_gap_block(input, ratio) sse = squeeze_excite_gmp_block(input, ratio) x = maximum([cse, sse]) #Lots of funsion can be done line average, max etc. return x
def create_model(): g = Input(shape=(DIMS, ), dtype=np.float, name='g') #lstm = LSTM(300)(g) #intermediate = Dense(DIMS)(g) lt = Dense(DIMS, activation='sigmoid', name='lt')(g) #lt = Lambda(normalize, name='lt')(lt_raw) dpos = Input(shape=(DIMS, ), dtype=np.float, name='dpos') dneg = Input(shape=(DIMS, ), dtype=np.float, name='dneg') ld = Dense(DIMS, activation='softmax', name='ld') ldpos = ld(dpos) ldneg = ld(dneg) dot = Dot(axes=1) lspos = dot([lt, ldpos]) lsneg = dot([lt, ldneg]) omega = Input(tensor=K.repeat_elements(K.constant([[1.0]]), BATCH_SIZE, 0), name='omega') zero = Input(tensor=K.repeat_elements(K.constant([[0.0]]), BATCH_SIZE, 0), name='zero') cost = maximum([zero, add([lsneg, subtract([omega, lspos])])], name='val_loss') #def loss(y_true, y_pred): # return y_pred model = Model(inputs=[g, dpos, dneg, omega, zero], outputs=[cost]) #optimizer = optimizers.Adam() # GRU, loss = 1199 #optimizer = optimizers.Adam(lr=0.0001) model.compile(optimizer='adam', loss='mean_squared_error', metrics=['mae', 'acc']) print(model.summary()) return model
def nn_model3(train_x,train_y): """建立第二个五层的神经网络""" inputs=Input(shape=(train_x.shape[1],)) x1 = Dense(40, activation='relu')(inputs) x1=BatchNormalization()(x1) x2 = Dense(40, activation='tanh')(inputs) x2=BatchNormalization()(x2) x=maximum([x1,x2]) x = Dense(20, activation='sigmoid')(x) predictions = Dense(10, activation='softmax')(x) model = Model(inputs=inputs, outputs=predictions) model.summary() model.compile(optimizer=#Adam(lr=0.001, epsilon=1e-09, decay=0.0), 'rmsprop', # "adam", loss='categorical_crossentropy', metrics=['accuracy']) model.fit(train_x, train_y,epochs=5, batch_size=500, validation_split=0.0) return model
def nn_model2(train_x, train_y): """建立第二个五层的神经网络""" inputs = Input(shape=(train_x.shape[1], )) x1 = Dense(500, activation='tanh')(inputs) x1 = bn_prelu(x1) x2 = Dense(500, activation='relu')(inputs) x2 = bn_prelu(x2) x = maximum([x1, x2]) x = Dense(50, activation='sigmoid')(x) x = bn_prelu(x) x = Dense(50, activation='relu')(x) x = Dense(50, activation='linear')(x) predictions = Dense(10, activation='softmax')(x) model = Model(inputs=inputs, outputs=predictions) model.compile(optimizer=Adam(lr=0.001, epsilon=1e-09, decay=0.0), loss='categorical_crossentropy', metrics=['accuracy']) model.fit(train_x, train_y, epochs=2, batch_size=200, validation_split=0.0) return model
def ensemble_models(models, input_shape, combination_mode='average', ensemble_name='ensemble'): ''' Args: models: List of keras models input_shape: Tuple containing input shape in tf format (H, W, C) combination_mode: The way probabilities will be joined. We support `average` and `maximum` ensemble_name: The name of the model that will be returned Returns: A model containing the ensemble of the `models` passed. Same `input_shape` will be used for all of them ''' if not len(input_shape) == 3: raise ValueError('Incorrect input shape, it should have 3 dimensions (H, W, C)') input_shape = Input(input_shape) combination_mode_options = ['average', 'maximum'] # Collect outputs of models in a list models_output = [] for i, model in enumerate(models): # Keras needs all the models to be named differently model.name = 'model_' + str(i) models_output.append(model(input_shape)) # Computing outputs if combination_mode in combination_mode_options: if combination_mode == 'average': out = average(models_output) elif combination_mode == 'maximum': out = maximum(models_output) # Build model from same input and outputs ensemble = Model(inputs=input_shape, outputs=out, name=ensemble_name) else: raise ValueError('Incorrect combination mode selected, we only allow for `average` or `maximum`') return ensemble
def create_model(nb_feats=25, emat=embedding_matrix): """Add little noise at MLP layer""" VOCAB = len(word2ix) EMBED_HIDDEN_SIZE = 300 MAX_LEN = 35 MAX_CHARLEN = 5 SENT_HIDDEN_SIZE = 100 ACTIVATION = 'elu' RNN_HIDDEN_SIZE = 50 DP = 0.25 L2 = 4e-6 embed_word = Embedding(VOCAB, EMBED_HIDDEN_SIZE, weights=[emat], input_length=MAX_LEN, trainable=False) embed_code = Embedding(len(code2Idx), len(code2Idx), input_length=MAX_LEN, trainable=True) translate = TimeDistributed( Dense(units=SENT_HIDDEN_SIZE, activation=ACTIVATION)) encode = Bidirectional(recurrent.LSTM(units=RNN_HIDDEN_SIZE, return_sequences=False, kernel_initializer='glorot_uniform', dropout=DP, recurrent_dropout=DP), name='my_lstm') # input defined: 8 tensors seq_title = Input(shape=(MAX_LEN, ), dtype='int32') # title seq_title_code = Input(shape=(MAX_LEN, ), dtype='int32') seq_title_char = Input(shape=(MAX_LEN, MAX_CHARLEN), dtype='int32') seq_cat = Input(shape=(MAX_LEN, ), dtype='int32') # joint cats seq_cat_code = Input(shape=(MAX_LEN, ), dtype='int32') seq_cat_char = Input(shape=(MAX_LEN, MAX_CHARLEN), dtype='int32') dense_input = Input(shape=(nb_feats, ), dtype='float32') # char charem_full = create_charem() # rnn encode seq = embed_word(seq_title) seq = Dropout(DP)(seq) seq = translate(seq) code = embed_code(seq_title_code) char = charem_full(seq_title_char) seq = concatenate([seq, code, char]) seq = encode(seq) seq3 = embed_word(seq_cat) seq3 = Dropout(DP)(seq3) seq3 = translate(seq3) code3 = embed_code(seq_cat_code) char3 = charem_full(seq_cat_char) seq3 = concatenate([seq3, code3, char3]) seq3 = encode(seq3) # dense den = BatchNormalization()(dense_input) den = Dense(100, activation=ACTIVATION)(den) den = Dropout(DP)(den) #joint1: LOGLOSS vs RMSE joint = concatenate([seq, seq3, den]) joint = Dense(units=150, activation=ACTIVATION, kernel_regularizer=l2(L2) if L2 else None, kernel_initializer='he_normal')(joint) joint = PReLU()(joint) joint = Dropout(DP)(joint) joint = BatchNormalization()(joint) joint = maximum([ Dense(units=100, activation=ACTIVATION, kernel_regularizer=l2(L2) if L2 else None, kernel_initializer='he_normal')(joint) for _ in range(5) ]) joint = PReLU()(joint) joint = Dropout(DP)(joint) joint = BatchNormalization()(joint) score1 = Dense(units=1, activation='sigmoid', kernel_regularizer=l2(L2) if L2 else None, kernel_initializer='he_normal', name='logloss')(joint) score2 = Dense(units=1, activation='sigmoid', kernel_regularizer=l2(L2) if L2 else None, kernel_initializer='he_normal', name='mse')(joint) # plug all in one model2 = Model(inputs=[ seq_title, seq_title_code, seq_title_char, seq_cat, seq_cat_code, seq_cat_char, dense_input ], outputs=[score1, score2]) model2.compile(optimizer='nadam', loss={'logloss': 'binary_crossentropy', 'mse': 'mean_squared_error'}, \ loss_weights={'logloss': 0.5, 'mse': 0.5}, metrics=[rmse_keras]) return model2
strides=(1, 1), padding='valid')(conv_0) #conv_1 = Conv2D(filters1, kernel_size=(filter_sizes[0], 100), padding='valid', kernel_initializer='normal', activation=newacti)(maxpool_0) #maxpool_1 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] +1, 1), strides=(1,1), padding='valid')(conv_1) gru = Reshape((sequence_length, embedding_vecor_length))(e) gru = GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(gru) gru = GRU(gru_output_size)(gru) gru = Reshape((1, 1, gru_output_size))(gru) merge = maximum([maxpool_0, gru]) flatten = Flatten()(merge) dropout = Dropout(0.5)(flatten) output = Dense(nclass, activation='softmax')(dropout) model = Model(inputs=visible, outputs=output) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) model.fit(padded_docs, y_train, epochs=1, verbose=0, nb_epoch=3, batch_size=64, validation_data=(vpadded_docs, y_valid))
STOP_LAYER = 101 # Form CNN1 and truncate after 101 layers cnn1 = applications.ResNet50(include_top=False, input_shape=(224, 224, 3)) cnn1 = Model(cnn1.input, cnn1.layers[STOP_LAYER].output) # Create 12 Input instances inputs = [] processed_inputs = [] for i in range(0, 12): temp = Input(shape=(224, 224, 3), name='input_' + str(i)) inputs.append(temp) processed_inputs.append(cnn1(temp)) # Get maximum tensor max = layers.maximum(processed_inputs) # Create 2 pairs of a convultional layer and a batch normalization layer x = Conv2D(filters=16, kernel_size=[5, 5], input_shape=(224, 224, 3))(max) x = BatchNormalization()(x) x = Conv2D(filters=16, kernel_size=[5, 5], input_shape=(224, 224, 3))(x) x = BatchNormalization()(x) x = (Flatten(input_shape=(224, 224, 3)))(x) x = Dense(40, activation='softmax')(x) model = Model(input=inputs, output=x) # Set the first p fraction of layers to non-trainable p = 0.7
img1 = Input(shape=(299, 299, 3), name='img_1') feature1 = base_model1(img1) feature2 = base_model2(img1) # let's add a fully-connected layer category_predict1 = Dense(100, activation='softmax', name='ctg_out_1')(Dropout(0.5)(feature1)) category_predict2 = Dense(100, activation='softmax', name='ctg_out_2')(Dropout(0.5)(feature2)) category_predict = Dense(100, activation='softmax', name='ctg_out')(concatenate([feature1, feature2])) max_category_predict = maximum([category_predict1, category_predict2]) model = Model(inputs=[img1], outputs=[ category_predict1, category_predict2, category_predict, max_category_predict ]) # model.save('dog_xception.h5') plot_model(model, to_file='single_model.png') # first: train only the top layers (which were randomly initialized) # i.e. freeze all convolutional InceptionV3 layers for layer in base_model1.layers: layer.trainable = False for layer in base_model2.layers:
def get_two_path_cascade_mf(stream_model, n_feature_maps=2, mode=None, summary=False, maxout=False, dropout=False): K.clear_session() k_f = PATCH_HEIGHT - k_g + 1 stream_model = stream_model(2 * PATCH_HEIGHT - d_mf, PATCH_HEIGHT - d_mf, mode=mode, n_feature_maps=n_feature_maps, maxout=maxout, dropout=dropout) stream_model.trainable = False stream_model.load_weights( f'mf_stream_{mode}.h5' if mode is not None else 'mf_stream.h5') if mode in ['ct', 'pet']: x = Input(shape=(2 * PATCH_HEIGHT - d_mf, 2 * PATCH_WIDTH - d_mf, 1)) model_input = x stream_output = stream_model(x) else: ct_input = Input(shape=(2 * PATCH_HEIGHT - d_mf, 2 * PATCH_WIDTH - d_mf, 1)) pet_input = Input(shape=(2 * PATCH_HEIGHT - d_mf, 2 * PATCH_WIDTH - d_mf, 1)) model_input = [ct_input, pet_input] x = concatenate(model_input, axis=-1) stream_output = stream_model([ct_input, pet_input]) h = (PATCH_HEIGHT - d_mf) // 2 w = (PATCH_WIDTH - d_mf) // 2 trim = d_mf % 2 == 1 x = Lambda(lambda x: x[:, h + trim:-h, w + trim:-w, :])(x) if maxout: conv1_local = maximum( [Conv2D(64, (k_1, k_1))(x) for _ in range(n_feature_maps)]) else: conv1_local = Conv2D(64, (k_1, k_1), activation='relu')(x) pool1_local = MaxPooling2D((p_1, p_1), strides=(1, 1))(conv1_local) if dropout: pool1_local = Dropout(0.2)(pool1_local) if maxout: conv2_local = maximum([ Conv2D(64, (k_2, k_2))(pool1_local) for _ in range(n_feature_maps) ]) else: conv2_local = Conv2D(64, (k_2, k_2), activation='relu')(pool1_local) pool2_local = MaxPooling2D((p_2, p_2), strides=(1, 1))(conv2_local) if dropout: pool2_local = Dropout(0.2)(pool2_local) if maxout: conv1_global = maximum( [Conv2D(160, (k_g, k_g))(x) for _ in range(n_feature_maps)]) else: conv1_global = Conv2D(160, (k_g, k_g), activation='relu')(x) if dropout: conv1_global = Dropout(0.2)(conv1_global) combine = concatenate([pool2_local, conv1_global, stream_output], axis=-1) output = Conv2D(1, (k_f, k_f), activation='sigmoid')(combine) output = Reshape((1, ))(output) model = Model(inputs=model_input, outputs=output) model.compile(optimizer=SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True), loss='binary_crossentropy', metrics=['accuracy']) if summary: model.summary() return model
def get_stream_model(big_patch_dim, small_patch_dim, n_feature_maps=2, mode=None, summary=False, maxout=False, dropout=False): K.clear_session() k_f = big_patch_dim - k_g + 1 if mode in ['ct', 'pet']: x = Input(shape=(big_patch_dim, big_patch_dim, 1)) model_input = x else: ct_input = Input(shape=(big_patch_dim, big_patch_dim, 1)) pet_input = Input(shape=(big_patch_dim, big_patch_dim, 1)) model_input = [ct_input, pet_input] x = concatenate(model_input, axis=-1) if maxout: conv1_local = maximum( [Conv2D(64, (k_1, k_1))(x) for _ in range(n_feature_maps)]) else: conv1_local = Conv2D(64, (k_1, k_1), activation='relu')(x) pool1_local = MaxPooling2D((p_1, p_1), strides=(1, 1))(conv1_local) if dropout: pool1_local = Dropout(0.2)(pool1_local) if maxout: conv2_local = maximum([ Conv2D(64, (k_2, k_2))(pool1_local) for _ in range(n_feature_maps) ]) else: conv2_local = Conv2D(64, (k_2, k_2), activation='relu')(pool1_local) pool2_local = MaxPooling2D((p_2, p_2), strides=(1, 1))(conv2_local) if dropout: pool2_local = Dropout(0.2)(pool2_local) if maxout: conv1_global = maximum( [Conv2D(160, (k_g, k_g))(x) for _ in range(n_feature_maps)]) else: conv1_global = Conv2D(160, (k_g, k_g), activation='relu')(x) if dropout: conv1_global = Dropout(0.2)(conv1_global) #combine = Flatten()(concatenate([pool2_local, conv1_global], axis=-1)) #output = Dense(small_patch_dim * small_patch_dim, activation='sigmoid')(combine) combine = concatenate([pool2_local, conv1_global], axis=-1) output = Conv2D(small_patch_dim * small_patch_dim, (k_f, k_f), activation='sigmoid')(combine) if small_patch_dim > 1: output = Reshape((small_patch_dim, small_patch_dim, 1))(output) else: output = Reshape((1, ))(output) model = Model(inputs=model_input, outputs=output) model.compile(optimizer=SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True), loss='binary_crossentropy', metrics=['accuracy']) if summary: model.summary() return model
def build_model(self, rnn_layer_sizes=np.array([20, 20, 20]), dense_layer_parallel_sizes=np.array([10, 20]), dense_layer_sequential_sizes=np.array([32, 20]), dropout_prob_rnn=0.3, dropout_prob_dense=0.3, lambda_reg_rnn=0.001, lambda_reg_dense=0.001, merge='multiply'): self.rnn_layer_sizes = rnn_layer_sizes self.dense_layer_parallel_sizes = dense_layer_parallel_sizes self.dense_layer_sequential_sizes = dense_layer_sequential_sizes self.dropout_prob_rnn = dropout_prob_rnn self.dropout_prob_dense = dropout_prob_dense self.lambda_reg_rnn = lambda_reg_rnn self.lambda_reg_dense = lambda_reg_dense self.merge = merge if dense_layer_parallel_sizes[-1] != rnn_layer_sizes[-1]: print( 'Dimensions of last layers of RNN and of parallel dense network must agree!' ) return # define inputs tracks_input = Input(self.input_shape_tracks, dtype='float32', name='tracks_input') session_input = Input(self.input_shape_sessions, dtype='float32', name='session_input') # RNN side x_rnn = LSTM(self.rnn_layer_sizes[0], return_sequences=True, kernel_regularizer=l2(self.lambda_reg_rnn))(tracks_input) for i in range(1, self.rnn_layer_sizes.size): x_rnn = LSTM(self.rnn_layer_sizes[i], return_sequences=True, kernel_regularizer=l2(self.lambda_reg_rnn))(x_rnn) x_rnn = BatchNormalization()(x_rnn) out_rnn = Dropout(self.dropout_prob_rnn)(x_rnn) # dense side x_fc = Dense(self.dense_layer_parallel_sizes[0], activation='relu', kernel_regularizer=l2( self.lambda_reg_dense))(session_input) for i in range(1, self.dense_layer_parallel_sizes.size): x_fc = Dense(self.dense_layer_parallel_sizes[i], activation='relu', kernel_regularizer=l2(self.lambda_reg_dense))(x_fc) x_fc = BatchNormalization()(x_fc) out_fc = Dropout(self.dropout_prob_dense)(x_fc) x = [] # merge RNN and dense side if self.merge == 'multiply': x = multiply([out_rnn, out_fc]) elif self.merge == 'add': out_fc = RepeatVector(20)(out_fc) x = add([out_rnn, out_fc]) elif self.merge == 'concatenate': out_fc = RepeatVector(20)(out_fc) x = concatenate([out_rnn, out_fc], axis=-1) elif self.merge == 'maximum': out_fc = RepeatVector(20)(out_fc) x = maximum([out_rnn, out_fc]) else: print( 'Choose proper merge variation: multiply, add, concatenate or maximum' ) for i in range(self.dense_layer_sequential_sizes.size - 1): x = Dense(self.dense_layer_sequential_sizes[i], activation='relu', kernel_regularizer=l2(self.lambda_reg_dense))(x) x = Dense(self.dense_layer_sequential_sizes[-1], activation='linear', kernel_regularizer=l2(self.lambda_reg_dense))(x) output = Reshape(self.output_shape, name='output')(x) # create model and compile it self.model = K_Model(inputs=[tracks_input, session_input], outputs=[output])
def build(self): print 'Latent dimensions: ' + str(self.latent_dim) encoders = [self.encoder_maker(m) for m in self.input_modalities] ind_emb = [lr for (input, lr) in encoders] self.org_ind_emb = [lr for (input, lr) in encoders] self.inputs = [input for (input, lr) in encoders] #apply spatial transformer # if self.spatial_transformer: # print 'Adding a spatial transformer layer' # input_shape = (self.latent_dim, self.H, self.W) # tpn = tpn_maker(input_shape) # mod1 = ind_emb[0] # aligned_ind_emb = [mod1] # for mod in ind_emb[1:]: # aligned_mod = merge([tpn([mod1, mod]), mod], mode=STMerge, output_shape=input_shape) # aligned_ind_emb.append(aligned_mod) # ind_emb = aligned_ind_emb if self.spatial_transformer: print 'Adding a spatial transformer layer' input_shape = (self.latent_dim, self.H, self.W) # input_shape = (self.H, self.W, self.latent_dim) tpn = tpn_maker(input_shape) mod1 = ind_emb[0] # mod1= K.permute_dimensions(ind_emb[0],(0,2,3,1)) aligned_ind_emb = [mod1] for mod in ind_emb[1:]: aligned_mod = merge([tpn([mod1, mod]), mod], mode=STMerge, output_shape=input_shape) aligned_ind_emb.append(aligned_mod) ind_emb = aligned_ind_emb if self.common_merge == 'hemis': self.all_emb = self.hemis(ind_emb) elif self.common_merge == 'cbam_block': weighted_rep = self.cbam_block(ind_emb, 'hjh') self.all_emb = ind_emb + [weighted_rep] else: assert self.common_merge == 'max' or self.common_merge == 'ave' or self.common_merge == 'rev_loss' print 'Fuse latent representations using ' + str(self.common_merge) if self.common_merge == 'ave': weighted_rep = average( ind_emb, name='combined_em') if len(self.inputs) > 1 else ind_emb[0] else: weighted_rep = maximum( ind_emb, name='combined_em') if len(self.inputs) > 1 else ind_emb[0] self.all_emb = ind_emb + [weighted_rep] self.decoders = [self.decoder_maker(m) for m in self.output_modalities] outputs = get_decoder_outputs(self.output_modalities, self.decoders, self.all_emb) # this is for minimizing the distance between the individual embeddings # outputs += self.get_embedding_distance_outputs(self.all_emb) print 'all outputs: ', [o.name for o in outputs] #out_dict = {'decoder_MASK' : mae } # out_dict = {'em_%d_dec_%s' % (emi, dec): mae for emi in range(self.num_emb) for dec in self.output_modalities} out_dict = {} out_dict['em_0_dec_FLAIR'] = mae2 out_dict['em_1_dec_FLAIR'] = mae2 out_dict['em_2_dec_FLAIR'] = mae2 # out_dict['em_3_dec_FLAIR'] = mae # out_dict['em_0_dec_MASK'] = dice_coef_loss # out_dict['em_1_dec_MASK'] = dice_coef_loss # out_dict['em_2_dec_MASK'] = dice_coef_loss_adj # out_dict['em_3_dec_MASK'] = dice_coef_loss # out_dict['em_0_dec_MASK'] = dice_loss # out_dict['em_1_dec_MASK'] = dice_loss # out_dict['em_2_dec_MASK'] = dice_loss # out_dict['em_3_dec_MASK'] = dice_loss_mask get_indiv_weight = lambda mod: self.output_weights[ mod] if self.ind_outs else 0.0 get_fused_weight = lambda mod: self.output_weights[ mod] if self.fuse_outs else 0.0 loss_weights = {} for dec in self.output_modalities: for emi in range(self.num_emb - 1): loss_weights['em_%d_dec_%s' % (emi, dec)] = get_indiv_weight(dec) loss_weights['em_%d_dec_%s' % (self.num_emb - 1, dec)] = get_fused_weight(dec) if len(self.inputs) > 1: if self.common_merge == 'rev_loss': out_dict['em_concat'] = mae # else: # out_dict['em_concat'] = embedding_distance # loss_weights['em_concat'] = self.output_weights['concat'] # out_dict['em_fused'] = embedding_distance # loss_weights['em_fused'] = self.output_weights['concat'] print 'output dict: ', out_dict print 'loss weights: ', loss_weights self.model = Model(input=self.inputs, output=outputs) self.model.summary() self.model.compile(optimizer=Adam(lr=0.0001), loss=out_dict, loss_weights=loss_weights)
conv_0 = Conv2D(filters1, kernel_size=(filter_sizes[0], 100), padding='valid', kernel_initializer='normal', activation=newacti)(e) maxpool_0 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] + 1, 1), strides=(1, 1), padding='valid')(conv_0) #maxpool_0=Flatten()(maxpool_0) #maxpool_0=Reshape((1,gru_output_size))(maxpool_0) gru = Reshape((sequence_length, embedding_vecor_length))(embedding) gru = GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(gru) merge2 = maximum([maxpool_0, gru]) merge = Reshape((sequence_length, filters1))(merge2) gru1 = GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(merge) #gru1=Flatten()(gru1) gru1 = MaxPooling1D(pool_size=8)(gru1) conv_1 = Conv1D(filters1, kernel_size, padding='valid', activation=newacti, strides=1)(merge) #gru2=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(conv_1)
name='xception') base_model2 = InceptionV3(include_top=True, weights='imagenet') base_model2 = Model(inputs=[base_model2.input], outputs=[base_model2.get_layer('avg_pool').output], name='inceptionv3') img1 = Input(shape=(224, 224, 3), name='img_1') feature1 = base_model1(img1) feature2 = base_model2(img1) categorical_predic1 = Dense(100, activation='softmax', name='ctg_out_1')(Dropout(0.5)(feature1)) categorical_predic2 = Dense(100, activation='softmax', name='ctg_out_2')(Dropout(0.5)(feature2)) categorcial_predict = Dense(100, activation='softmax', name='ctg_out')(concatenate([feature1, feature2])) max_category_predict = maximum([categorical_predic1, categorical_predic2]) model = Model(inputs=[img1], outputs=[ categorical_predic1, categorical_predic2, categorcial_predict, max_category_predict ]) print(model.summary())
#maxpool_1 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] +1, 1), strides=(1,1), padding='valid')(conv_1) gru = Reshape((sequence_length, embedding_vecor_length))(e) gru = GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(gru) #gru=Reshape((1,gru_output_size))(gru) #gru=RepeatVector(sequence_length)(gru) gru = Conv1D(64, kernel_size, padding='valid', activation=newacti, strides=1)(gru) gru = MaxPooling1D(pool_size=pool_size)(gru) #gru=Reshape((1,1,gru_output_size))(gru) merge = maximum([maxpool_0, gru]) conv_1 = Conv1D(64, kernel_size, padding='valid', activation=newacti, strides=1)(merge) maxpool_1 = MaxPooling1D(pool_size=pool_size)(conv_1) #conv_1 = Conv2D(filters1, kernel_size=(filter_sizes[0], 100), padding='valid', kernel_initializer='normal', activation=newacti)(maxpool_0) #maxpool_1 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] +1, 1), strides=(1,1), padding='valid')(conv_1) #gru=Reshape((sequence_length,embedding_vecor_length))(merge) gru1 = GRU(gru_output_size, return_sequences=True,
#define network input = Input(shape=input_shape) #hidden layer 0 left = Conv2D(filters=96, kernel_size=(8, 8), activation='relu', padding='same', name='h0_l')(input) right = Conv2D(filters=96, kernel_size=(8, 8), activation='relu', padding='same', name='h0_r')(input) h0 = maximum([left, right]) #apply max pooling h0 = MaxPooling2D(pool_size=(4, 4), strides=(2, 2), name='pool0')(h0) #hidden layer 1 left = Conv2D(filters=192, kernel_size=(8, 8), activation='relu', padding='same', name='h1_l')(h0) right = Conv2D(filters=192, kernel_size=(8, 8), activation='relu', padding='same', name='h1_r')(h0) h1 = maximum([left, right])