def unet_resnet101(img_rows, img_cols, color_type, num_classes=1): encode_model, layers = resnet101_model(img_rows, img_cols, color_type) input = encode_model.input # layer1 = encode_model.get_layer('conv1_relu') # layer2 = encode_model.get_layer('res2c_relu') # layer3 = encode_model.get_layer('res3b2_relu') # layer4 = encode_model.get_layer('res4b22_relu') layer1, layer2, layer3, layer4 = layers x = encode_model.output x = upsample(x, layer4, 1024 // 2) x = upsample(x, layer3, 512 // 2) x = upsample(x, layer2, 256 // 2) x = upsample(x, layer1, 128 // 2) x = upsample(x, input, 64 // 2) output1 = Conv2D(num_classes, (1, 1), activation='sigmoid')(x) if not USE_REFINE_NET: model = Model(inputs=input, outputs=[output1]) # model.load_weights('../weights/head-segmentation-model.h5') # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss='binary_crossentropy', metrics=[dice_loss]) return model else: inputs2 = concatenate([input, output1]) outputs2 = block2(inputs2, 64) outputs2 = average([output1, outputs2]) model2 = Model(inputs=input, outputs=[output1, outputs2]) return model2
def getModel(): inputShape = (32, 128, 1) rnnUnits = 256 maxStringLen = 32 inputs = Input(name='inputX', shape=inputShape, dtype='float32') inner = res_block(inputs, 64) inner = res_block(inner, 64) inner = MaxPooling2D(pool_size=(2, 2), name='MaxPoolName1')(inner) inner = res_block(inner, 128) inner = res_block(inner, 128) inner = MaxPooling2D(pool_size=(2, 2), name='MaxPoolName2')(inner) inner = res_block(inner, 256) inner = res_block(inner, 256) inner = MaxPooling2D(pool_size=(1, 2), strides=(2, 2), name='MaxPoolName4')(inner) inner = res_block(inner, 512) inner = res_block(inner, 512) inner = MaxPooling2D(pool_size=(1, 2), strides=(2, 2), name='MaxPoolName6')(inner) inner = res_block(inner, 512) inner = Reshape(target_shape=(maxStringLen, rnnUnits), name='reshape')(inner) LSF = LSTM(rnnUnits, return_sequences=True, kernel_initializer='he_normal', name='LSTM1F')(inner) LSB = LSTM(rnnUnits, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='LSTM1B')(inner) LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB) LS1 = average([LSF, LSB]) LS1 = BatchNormalization()(LS1) LSF = LSTM(rnnUnits, return_sequences=True, kernel_initializer='he_normal', name='LSTM2F')(LS1) LSB = LSTM(rnnUnits, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='LSTM2B')(LS1) LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB) LS2 = concatenate([LSF, LSB]) LS2 = BatchNormalization()(LS2) yPred = Dense(len(unicodes) + 1, kernel_initializer='he_normal', name='dense2')(LS2) yPred = Activation('softmax')(yPred) return Model(inputs=[inputs], outputs=yPred)
def get_csv_plus_image_model(network, num_classes, features, img_width, img_height, merge_type='concat'): """Returns a model that uses images and csv as Input Arguments: network {string} -- Name of the network to use pretrained on imagenet num_classes {int} -- Number of classes in the dataset features {int} -- Number of features used from csv data img_width {int} -- Image width img_height {int} -- Image height Keyword Arguments: merge_type {str} -- [Type of merge process for inner neural networks] (default: {'concat'}) Returns: [tuple] -- Keras Model with dual input and last layer number from cnn part. """ input_shape = (img_width, img_height, 3) image_input = Input(shape=input_shape) base_model, last_layer_number = get_cnn_model(network, input_shape, image_input) x = base_model.output x = GlobalAveragePooling2D()(x) x = Dense(512, activation='relu')(x) # Create MLP using features from csv files aux_input = Input(shape=(features, )) aux = Dense(512, activation='relu')(aux_input) aux = Dropout(0.3)(aux) aux = Dense(512, activation='relu')(aux) aux = Dropout(0.3)(aux) aux = Dense(512, activation='relu')(aux) # Merge both inputs if merge_type == 'concat': merge = concatenate([x, aux]) elif merge_type == 'add': merge = add([x, aux]) elif merge_type == 'mul': merge = multiply([x, aux]) elif merge_type == 'avg': merge = average([x, aux]) predictions = Dense(num_classes, activation='softmax')(merge) return Model(inputs=[base_model.input, aux_input], output=predictions), last_layer_number
def call(self, xs, mask=None): assert len(xs) == 2 # separate out input matrices # x1, x2: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE) x1, x2 = xs # build alignment matrix # alpha: (BATCH_SIZE, MAX_TIMESTEPS, MAX_TIMESTEPS) alpha = K.softmax( K.batch_dot(K.dot(x2, self.W), K.permute_dimensions(x1, (0, 2, 1)))) # build context vectors # c1, c2: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE) c1 = K.repeat_elements(K.sum(K.batch_dot(alpha, x2), axis=1, keepdims=True), self.max_timesteps, axis=1) c2 = K.repeat_elements(K.sum(K.batch_dot( K.permute_dimensions(alpha, (0, 2, 1)), x1), axis=1, keepdims=True), self.max_timesteps, axis=1) # build attention vector # o1t, o2t: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE) o1t = K.tanh(K.dot(K.concatenate([c1, x1], axis=2), self.U1)) o2t = K.tanh(K.dot(K.concatenate([c2, x2], axis=2), self.U2)) # masking if mask is not None and mask[0] is not None: o1t *= K.cast( K.repeat_elements(K.expand_dims(mask[0], axis=2), o1t.shape[2], 2), K.floatx()) if mask is not None and mask[1] is not None: o2t *= K.cast( K.repeat_elements(K.expand_dims(mask[0], axis=2), o2t.shape[2], 2), K.floatx()) # sum over timesteps # o1, o2: (BATCH_SIZE, EMBED_SIZE) o1 = K.sum(o1t, axis=1) o2 = K.sum(o2t, axis=1) # merge the attention vectors according to merge_mode if self.merge_mode == "concat": return concatenate([o1, o2], axis=1) elif self.merge_mode == "diff": return add([o1, -o2]) elif self.merge_mode == "prod": return multiply([o1, o2]) elif self.merge_mode == "avg": return average([o1, o2]) else: # max return maximum([o1, o2])
def net(channel_axis): # Determine proper input shape in_shape = [(24, 42, 42, 1), (42, 24, 42, 1), (42, 42, 24, 1)] strides = [(1, 2, 2), (2, 1, 2), (2, 2, 1)] kernels = [(5, 5, 5), (5, 5, 5), (5, 5, 5)] dropout_conv = .2 dropout_dence = .2 inputs = [keras.layers.Input(shape=inx) for inx in in_shape] coders = [ _encoder(in_tensor=tnsr, stride=stride, bn_axis=channel_axis, kernel=kernel, dropout=dropout_conv) for tnsr, stride, kernel in zip(inputs, strides, kernels) ] x = average(coders) # shape: 128, 9, 10, 10 x = _conv_block(x, 3, [128, 128], dropout=dropout_conv, bn_axis=channel_axis) x = _identity_block(x, 3, [128, 128], dropout=dropout_conv, bn_axis=channel_axis) x = _conv_block(x, 3, [256, 256], stride=(2, 2, 2), dropout=dropout_conv, bn_axis=channel_axis) x = _identity_block(x, 3, [256, 256], dropout=dropout_conv, bn_axis=channel_axis) x = keras.layers.BatchNormalization(axis=channel_axis)(x) x = keras.layers.Activation('relu')(x) x = keras.layers.AveragePooling3D((2, 2, 2))(x) x = keras.layers.Flatten()(x) x = keras.layers.Dropout(dropout_dence)(x) x = keras.layers.Dense(256)(x) x = keras.layers.LeakyReLU(.3)(x) x = keras.layers.Dropout(dropout_dence)(x) x = keras.layers.Dense(2, activation='softmax', name='is_nodule')(x) model = keras.models.Model(inputs, x) model.compile('adam', 'categorical_crossentropy') return model
def show_basic_eval_for_snapshot_combo(snapshot_weight_list, scratch_model_func, weight_dirname, test_data): # Seperate test_data x_test, y_test = test_data nb_classes = y_test.shape[1] # Create Snapshot Ensemble model chkpt_models = [] for i, filename in enumerate(snapshot_weight_list): model = scratch_model_func() model.load_weights(weight_dirname + filename) for j, cur_layer in enumerate(model.layers): cur_layer.name = ("tmpy_%d_%d" % (i, j)) chkpt_models.append(model) chkpt_inputs = [model.input for model in chkpt_models] chkpt_outputs = [model.output for model in chkpt_models] if len(chkpt_models) > 1: final_output = average(chkpt_outputs) else: final_output = chkpt_outputs[0] snapshot_model = Model(inputs=chkpt_inputs, outputs=final_output) snapshot_model.compile( optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', metrics.top_k_categorical_accuracy]) # Display eval metrics snapshot_size = len(snapshot_weight_list) print("\nEvaluating the %d model shotshot" % snapshot_size) print(snapshot_weight_list) eval_metrics = snapshot_model.evaluate([x_test] * snapshot_size, y_test, batch_size=int( math.ceil(48 / snapshot_size)), verbose=1) print(snapshot_model.metrics_names) print(eval_metrics) # Return nothing return None
def Model_sent2tag_MLP_1(sentvocabsize, tagvocabsize, sent_W, tag_W, s2v_k, tag2v_k): input_sent = Input(shape=(1, ), dtype='int32') sent_embedding = Embedding(input_dim=sentvocabsize, output_dim=s2v_k, input_length=1, mask_zero=False, trainable=False, weights=[sent_W])(input_sent) input_tag = Input(shape=(1, ), dtype='int32') tag_embedding = Embedding(input_dim=tagvocabsize, output_dim=tag2v_k, input_length=1, mask_zero=False, trainable=False, weights=[tag_W])(input_tag) x1_1 = Flatten()(sent_embedding) x2_0 = Flatten()(tag_embedding) # x1_1 = Dense(100, activation='tanh')(x1_0) sub = subtract([x2_0, x1_1]) mul = multiply([x2_0, x1_1]) max = maximum([x2_0, x1_1]) avg = average([x2_0, x1_1]) class_input = concatenate([x2_0, x1_1, sub, mul, max, avg], axis=-1) # class_input = Flatten()(class_input) class_mlp1 = Dense(200, activation='tanh')(class_input) class_mlp1 = Dropout(0.5)(class_mlp1) class_mlp2 = Dense(2)(class_mlp1) class_output = Activation('softmax', name='CLASS')(class_mlp2) # distance = Lambda(euclidean_distance, output_shape=eucl_dist_output_shape)([mlp_x1_2, x2_0]) # distance = dot([x1_0, x2_0], axes=-1, normalize=True) mymodel = Model([input_sent, input_tag], class_output) mymodel.compile(loss='categorical_crossentropy', optimizer=optimizers.Adam(lr=0.001), metrics=['acc']) return mymodel
def construct_model(self): input1 = Input(shape=(225, 225, 1)) input2 = Input(shape=(225, 225, 1)) input3 = Input(shape=(225, 225, 1)) input4 = Input(shape=(225, 225, 1)) input5 = Input(shape=(225, 225, 1)) y1 = build_model(input1) y2 = build_model(input2) y3 = build_model(input3) y4 = build_model(input4) y5 = build_model(input5) out = merge.average([y1, y2, y3, y4, y5]) # out = keras.layers.concatenate([y1, y2], axis=0) self.model = Model(inputs=[input1, input2, input3, input4, input5], outputs=out) print(self.model.summary())
def get_res_layer2(input, output): ''' 26-30个残差块 ''' dense_res11 = Dense(20, activation='selu', kernel_initializer='lecun_normal')(output) dense_res12 = Dense(24, activation='linear', kernel_initializer='lecun_normal')(dense_res11) dense_res21 = Dense(20, activation='selu', kernel_initializer='lecun_normal')(dense_res12) dense_res22 = Dense(24, activation='linear', kernel_initializer='lecun_normal')(dense_res21) dense_res31 = Dense(20, activation='selu', kernel_initializer='lecun_normal')(dense_res22) dense_res32 = Dense(24, activation='linear', kernel_initializer='lecun_normal')(dense_res31) dense_res41 = Dense(20, activation='selu', kernel_initializer='lecun_normal')(dense_res32) dense_res42 = Dense(24, activation='linear', kernel_initializer='lecun_normal')(dense_res41) dense_res51 = Dense(20, activation='selu', kernel_initializer='lecun_normal')(dense_res42) dense_res52 = Dense(24, activation='linear', kernel_initializer='lecun_normal')(dense_res51) output_new = average([dense_res52, input, output]) return output_new
def _create_model(self, shape): n_objects, n_features = shape[1].value, shape[2].value if hasattr(self, "n_features"): if self.n_features != n_features: logger.error("Number of features is not consistent.") input_layer = Input(shape=(n_objects, n_features)) inputs = [create_input_lambda(i)(input_layer) for i in range(n_objects)] # Connect input tensors with set mapping layer: set_mappings = [] for i in range(n_objects): curr = inputs[i] for j in range(len(self.set_mapping_layers)): curr = self.set_mapping_layers[j](curr) set_mappings.append((i, curr)) # TODO: is feature_repr used outside? feature_repr = average([x for (j, x) in set_mappings]) self.cached_models[n_objects] = Model(inputs=input_layer, outputs=feature_repr)
def call(self, xs, mask=None): assert len(xs) == 2 # separate out input matrices # x1.shape == (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE) # x2.shape == (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE) x1, x2 = xs # build alignment matrix alpha = K.softmax(K.batch_dot(x1, x2, axes=(2, 2))) # align inputs # a1t, a2t: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE) a1t = K.batch_dot(alpha, x2, axes=(1, 1)) a2t = K.batch_dot(alpha, x1, axes=(2, 1)) # produce aligned outputs # o1t, o2t: (BATCH_SIZE, MAX_TIMESTEPS*2, EMBED_SIZE) o1t = K.tanh(K.dot(x1, self.U1) + K.dot(a1t, self.V1)) o2t = K.tanh(K.dot(x2, self.U2) + K.dot(a2t, self.V2)) # masking if mask is not None and mask[0] is not None: o1t *= K.cast( K.repeat_elements(K.expand_dims(mask[0], axis=2), o1t.shape[2], 2), K.floatx()) if mask is not None and mask[1] is not None: o2t *= K.cast( K.repeat_elements(K.expand_dims(mask[1], axis=2), o2t.shape[2], 2), K.floatx()) # o1, o2: (BATCH_SIZE, EMBED_SIZE) o1 = K.mean(o1t, axis=1) o2 = K.mean(o2t, axis=1) # merge the attention vectors according to merge_mode if self.merge_mode == "concat": return concatenate([o1, o2], axis=1) elif self.merge_mode == "diff": return add([o1, -o2]) elif self.merge_mode == "prod": return multiply([o1, o2]) elif self.merge_mode == "avg": return average([o1, o2]) else: # max return maximum([o1, o2])
def __init__(self, unit=64, dropout=0.2, max_len=39, update_num=3, regularization=0.1, embedding_matrix=None, use_cudnn=False, use_share=False, use_one_cell=False): self.unit = unit self.dropout = dropout self.use_share = use_share self.use_one_cell = use_one_cell self.regularization = l2(regularization) Q1_input = Input(shape=(max_len, ), dtype='int32', name='Q1') # (?, L) Q2_input = Input(shape=(max_len, ), dtype='int32', name='Q2') # (?, L) # Q1_m = Input(shape=(max_len,), dtype='int32', name='mask1') # Q2_m = Input(shape=(max_len,), dtype='int32', name='mask2') # magic = Input(shape=(4,), dtype='float32', name='magic') embedding = Embedding(input_dim=embedding_matrix.shape[0], output_dim=embedding_matrix.shape[1], mask_zero=True, weights=[embedding_matrix], trainable=False) # bn = BatchNormalization() Q1 = embedding(Q1_input) Q2 = embedding(Q2_input) GRULayer = CuDNNGRU if use_cudnn else GRU for i in range(update_num): Q1, Q2 = self.update_module(Q1, Q2, GRULayer) Q1, Q2 = self.attention(Q1, Q2, GRULayer, implementation=3) # bn1 = BatchNormalization() # bnm = BatchNormalization() # bns = BatchNormalization() # regression = Bilinear(implementation=0, activation='tanh')([Q1, Q2]) att = SelfAttention(1, activation='tanh') Q1 = att(Q1) Q2 = att(Q2) vector = concatenate([ # Q1, Q2, merge.multiply([Q1, Q2]), # merge.subtract([Q1, Q2], use_abs=True), merge.subtract([Q1, Q2]), merge.average([Q1, Q2]) ]) # vector = merge.subtract([Q1, Q2]) # vector = merge.add([Q1, Q2]) # vector = Dropout(self.dropout)(vector) # vector = Dense(units=512, activation='tanh')(vector) # magic_new = Dense(units=64, activation='tanh')(magic) # vector = concatenate([vector, magic_new]) vector = Dropout(self.dropout)(vector) vector = Dense(units=256, activation='tanh')(vector) vector = Dropout(self.dropout)(vector) regression = Dense(units=1, activation='sigmoid')(vector) super(IAM, self).__init__(inputs=[Q1_input, Q2_input], outputs=regression)
def Model_BiLSTM_CRF_multi3_1(sourcevocabsize, targetvocabsize, source_W, input_seq_lenth, output_seq_lenth, hidden_dim, emd_dim, sourcecharsize, character_W, input_word_length, char_emd_dim, targetpossize, batch_size=32, loss='categorical_crossentropy', optimizer='rmsprop'): word_input = Input(shape=(input_seq_lenth, ), dtype='int32') char_input = Input(shape=( input_seq_lenth, input_word_length, ), dtype='int32') char_embedding = Embedding(input_dim=sourcecharsize, output_dim=char_emd_dim, batch_input_shape=(batch_size, input_seq_lenth, input_word_length), mask_zero=False, trainable=True, weights=[character_W]) char_embedding2 = TimeDistributed(char_embedding)(char_input) char_cnn = TimeDistributed( Conv1D(50, 3, activation='relu', padding='valid'))(char_embedding2) char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn) char_macpool = Dropout(0.5)(char_macpool) word_embedding = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth, mask_zero=False, trainable=True, weights=[source_W])(word_input) word_embedding_dropout = Dropout(0.5)(word_embedding) embedding = concatenate([word_embedding_dropout, char_macpool], axis=-1) BiLSTM = Bidirectional(LSTM( hidden_dim, return_sequences=True, ), merge_mode='concat')(embedding) # BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True))(word_embedding_dropout) BiLSTM = BatchNormalization(axis=1)(BiLSTM) BiLSTM_dropout = Dropout(0.5)(BiLSTM) mlp3_hidden1 = Bidirectional(LSTM(hidden_dim, return_sequences=True), merge_mode='concat')(BiLSTM_dropout) mlp3_hidden1 = Dropout(0.5)(mlp3_hidden1) mlp3_hidden2 = TimeDistributed(Dense(100, activation='tanh'))(mlp3_hidden1) output3 = TimeDistributed(Dense(targetpossize + 1, activation='softmax'), name='pos')(mlp3_hidden2) embedding2 = concatenate([BiLSTM_dropout, mlp3_hidden1], axis=-1) mlp1_hidden1 = Bidirectional(LSTM(hidden_dim, return_sequences=True), merge_mode='concat')(embedding2) mlp1_hidden1 = Dropout(0.5)(mlp1_hidden1) mlp1_hidden2 = TimeDistributed(Dense(100, activation='tanh'))(mlp1_hidden1) # output1 = TimeDistributed(Dense(5+1, activation='softmax'), name='BIOES')(decodelayer1) mlp1_hidden3 = TimeDistributed(Dense(5 + 1, activation=None))(mlp1_hidden2) crflayer1 = CRF(5 + 1, sparse_target=False, learn_mode='marginal', name='BIOES') output1 = crflayer1(mlp1_hidden3) mlp2_hidden1 = Bidirectional(LSTM(hidden_dim, return_sequences=True), merge_mode='concat')(embedding2) mlp2_hidden1 = Dropout(0.5)(mlp2_hidden1) mlp2_hidden2 = TimeDistributed(Dense(100, activation='tanh'))(mlp2_hidden1) # output2 = TimeDistributed(Dense(5+1, activation='softmax'), name='Type')(decodelayer2) mlp2_hidden3 = TimeDistributed(Dense(5 + 1, activation=None))(mlp2_hidden2) mlp2_hidden4 = average([mlp1_hidden3, mlp2_hidden3]) crflayer2 = CRF(5 + 1, sparse_target=False, name='Type', learn_mode='marginal') output2 = crflayer2(mlp2_hidden4) Models = Model([word_input, char_input], [output1, output2, output3]) # Models.compile(optimizer=optimizers.RMSprop(lr=0.001), # loss={'finall': crflayer.loss_function, 'BIOES': 'categorical_crossentropy', 'Type': 'categorical_crossentropy'}, # loss_weights={'finall': 1., 'BIOES': 1., 'Type': 1.}, # metrics={'finall': [crflayer.accuracy], 'BIOES': ['acc'], 'Type': ['acc']}) Models.compile(optimizer=optimizers.RMSprop(lr=0.001), loss={ 'BIOES': crflayer1.loss_function, 'Type': crflayer2.loss_function, 'pos': 'categorical_crossentropy' }, loss_weights={ 'BIOES': 1., 'Type': 1., 'pos': 0.5 }, metrics={ 'BIOES': [crflayer2.accuracy], 'Type': [crflayer2.accuracy], 'pos': ['acc'] }) return Models
def model_mlp_mk_rff(input_dimensions, embedding_dim, hidden_classification_layers, n_classes): """Build a match kernel-like model that computes the average of RFF feature for a set and then classifies it using specified fully connected layers. Parameters ---------- input_dimensions : dict Dictionary of dimensions of the input features: {dim0: n_features0, dim1: n_features1, ...} embedding_dim : int Dimension of the RFF output space (approximation of the RKHS associated to the best kernel) hidden_classification_layers : list Number of units per hidden layer in the fully connected MLP classification part of the model n_classes : int Number of classes for the classification problem Returns ------- Model The full model (including the Logistic Regression step), but not compiled Examples -------- >>> m = model_mlp_mk_rff(input_dimensions={2: 5}, embedding_dim=10, hidden_classification_layers=[5, 5], n_classes=2) >>> m.count_params() 127 """ inputs = [] for d in sorted(input_dimensions.keys()): n_features = input_dimensions[d] inputs.extend([Input(shape=(d, )) for _ in range(n_features)]) if len(inputs) == 1: rff_layer = RFFLayer(units=embedding_dim) concatenated_avg_rffs = rff_layer(inputs[0]) elif len(input_dimensions) == 1: rff_layer = RFFLayer(units=embedding_dim) rffs = [rff_layer(input_feature) for input_feature in inputs] concatenated_avg_rffs = average(rffs) else: avg_rffs = [] idx0 = 0 rff_layers = {} for d in sorted(input_dimensions.keys()): n_features = input_dimensions[d] rff_layers[d] = RFFLayer(units=embedding_dim) rffs = [ rff_layers[d](input_feature) for input_feature in inputs[idx0:idx0 + n_features] ] avg_rffs.append(average(rffs)) idx0 += n_features concatenated_avg_rffs = concatenate(avg_rffs) hidden_layers = [concatenated_avg_rffs] for n_units in hidden_classification_layers: hidden_layers.append( Dense(units=n_units, activation="tanh")(hidden_layers[-1])) predictions = Dense(units=n_classes, activation="softmax")(hidden_layers[-1]) return Model(inputs=inputs, outputs=predictions)
def CompoundNet_VGG19(include_top=True, weights=None, input_tensor=None, input_shape=None, fusion_strategy='concatenate', mode='fine_tuning', pooling_mode='avg', classes=9, data_augm_enabled=False): """Instantiates the CompoundNet VGG19 architecture fine-tuned (2 steps) on Human Rights Archive dataset. Optionally loads weights pre-trained on Human Rights Archive Database. # Arguments include_top: whether to include the 3 fully-connected layers at the top of the network. weights: one of `None` (random initialization), 'HRA' (pre-training on Human Rights Archive), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or `(3, 224, 224)` (with `channels_first` data format). It should have exactly 3 input channels, and width and height should be no smaller than 48. E.g. `(200, 200, 3)` would be one valid value. fusion_strategy: one of `concatenate` (feature vectors of different sources are concatenated into one super-vector), `average` (the feature set is averaged) or `maximum` (selects the highest value from the corresponding features). mode: one of `feature_extraction` (freeze all but the penultimate layer and re-train the last Dense layer) or `fine_tuning` (unfreeze the lower convolutional layers and retrain more layers). pooling_mode: Optional pooling_mode mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling_mode will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling_mode will be applied. classes: optional number of classes to classify images into, only to be specified if `weights` argument is `None`. data_augm_enabled: whether to use the augmented samples during training. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`. """ if not (weights in {'HRA', None} or os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `HRA` ' '(pre-training on Human Rights Archive), ' 'or the path to the weights file to be loaded.') if not (fusion_strategy in {'concatenate', 'average', 'maximum'}): raise ValueError( 'The `fusion_strategy` argument should be either ' '`concatenate` (feature vectors of different sources are concatenated into one super-vector), ' '`average` (the feature set is averaged) ' 'or `maximum` (selects the highest value from the corresponding features).' ) if not (pooling_mode in {'avg', 'max', 'flatten'}): raise ValueError('The `pooling_mode` argument should be either ' '`avg` (GlobalAveragePooling2D), `max` ' '(GlobalMaxPooling2D), ' 'or `flatten` (Flatten).') if weights == 'HRA' and classes != 9: raise ValueError( 'If using `weights` as Human Rights Archive, `classes` should be 9.' ) cache_subdir = 'HRA_models' # Determine proper input shape input_shape = _obtain_input_shape(input_shape, default_size=224, min_size=48, data_format=K.image_data_format(), require_flatten=include_top, weights=weights) if input_tensor is None: img_input = Input(shape=input_shape) else: if not K.is_keras_tensor(input_tensor): img_input = Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor input_tensor = Input(shape=(224, 224, 3)) object_centric_model = VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False) scene_centric_model = VGG16_Places365(input_tensor=input_tensor, weights='places', include_top=False) # retrieve the ouputs object_model_output = object_centric_model.output scene_model_output = scene_centric_model.output # We will feed the extracted features to a merging layer if fusion_strategy == 'concatenate': merged = concatenate([object_model_output, scene_model_output]) elif fusion_strategy == 'average': merged = average([object_model_output, scene_model_output]) else: merged = maximum([object_model_output, scene_model_output]) if include_top: if pooling_mode == 'avg': x = GlobalAveragePooling2D(name='GAP')(merged) elif pooling_mode == 'max': x = GlobalMaxPooling2D(name='GMP')(merged) elif pooling_mode == 'flatten': x = Flatten(name='FLATTEN')(merged) x = Dense(256, activation='relu', name='FC1')(x) # let's add a fully-connected layer # When random init is enabled, we want to include Dropout, # otherwise when loading a pre-trained HRA model we want to omit # Dropout layer so the visualisations are done properly (there is an issue if it is included) if weights is None: x = Dropout(0.5, name='DROPOUT')(x) # and a logistic layer with the number of classes defined by the `classes` argument x = Dense(classes, activation='softmax', name='PREDICTIONS')(x) # new softmax layer # Ensure that the model takes into account any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # this is the transfer learning model we will train model = Model(inputs=inputs, outputs=x, name='CompoundNet-VGG19') # load weights if weights == 'HRA': if include_top: if mode == 'feature_extraction': for layer in object_centric_model.layers: layer.trainable = False for layer in scene_centric_model.layers: layer.trainable = False model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy') if data_augm_enabled: if fusion_strategy == 'concatenate': if pooling_mode == 'avg': weights_path = get_file( AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_AVG_POOL_fname, AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_FLATTEN_fname, AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_MAX_POOL_fname, AUGM_FEATURE_EXTRACTION_CONCATENATE_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif fusion_strategy == 'average': if pooling_mode == 'avg': weights_path = get_file( AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_AVG_POOL_fname, AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_FLATTEN_fname, AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_MAX_POOL_fname, AUGM_FEATURE_EXTRACTION_AVERAGE_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif fusion_strategy == 'maximum': if pooling_mode == 'avg': weights_path = get_file( AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_AVG_POOL_fname, AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_FLATTEN_fname, AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_MAX_POOL_fname, AUGM_FEATURE_EXTRACTION_MAXIMUM_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) else: if fusion_strategy == 'concatenate': if pooling_mode == 'avg': weights_path = get_file( FEATURE_EXTRACTION_CONCATENATE_FUSION_AVG_POOL_fname, FEATURE_EXTRACTION_CONCATENATE_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( FEATURE_EXTRACTION_CONCATENATE_FUSION_FLATTEN_fname, FEATURE_EXTRACTION_CONCATENATE_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( FEATURE_EXTRACTION_CONCATENATE_FUSION_MAX_POOL_fname, FEATURE_EXTRACTION_CONCATENATE_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif fusion_strategy == 'average': if pooling_mode == 'avg': weights_path = get_file( FEATURE_EXTRACTION_AVERAGE_FUSION_AVG_POOL_fname, FEATURE_EXTRACTION_AVERAGE_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( FEATURE_EXTRACTION_AVERAGE_FUSION_FLATTEN_fname, FEATURE_EXTRACTION_AVERAGE_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( FEATURE_EXTRACTION_AVERAGE_FUSION_MAX_POOL_fname, FEATURE_EXTRACTION_AVERAGE_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif fusion_strategy == 'maximum': if pooling_mode == 'avg': weights_path = get_file( FEATURE_EXTRACTION_MAXIMUM_FUSION_AVG_POOL_fname, FEATURE_EXTRACTION_MAXIMUM_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( FEATURE_EXTRACTION_MAXIMUM_FUSION_FLATTEN_fname, FEATURE_EXTRACTION_MAXIMUM_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( FEATURE_EXTRACTION_MAXIMUM_FUSION_MAX_POOL_fname, FEATURE_EXTRACTION_MAXIMUM_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif mode == 'fine_tuning': for layer in model.layers[:36]: layer.trainable = False for layer in model.layers[36:]: layer.trainable = True model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy') if data_augm_enabled: if fusion_strategy == 'concatenate': if pooling_mode == 'avg': weights_path = get_file( AUGM_FINE_TUNING_CONCATENATE_FUSION_AVG_POOL_fname, AUGM_FINE_TUNING_CONCATENATE_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( AUGM_FINE_TUNING_CONCATENATE_FUSION_FLATTEN_fname, AUGM_FINE_TUNING_CONCATENATE_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( AUGM_FINE_TUNING_CONCATENATE_FUSION_MAX_POOL_fname, AUGM_FINE_TUNING_CONCATENATE_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif fusion_strategy == 'average': if pooling_mode == 'avg': weights_path = get_file( AUGM_FINE_TUNING_AVERAGE_FUSION_AVG_POOL_fname, AUGM_FINE_TUNING_AVERAGE_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( AUGM_FINE_TUNING_AVERAGE_FUSION_FLATTEN_fname, AUGM_FINE_TUNING_AVERAGE_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( AUGM_FINE_TUNING_AVERAGE_FUSION_MAX_POOL_fname, AUGM_FINE_TUNING_AVERAGE_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif fusion_strategy == 'maximum': if pooling_mode == 'avg': weights_path = get_file( AUGM_FINE_TUNING_MAXIMUM_FUSION_AVG_POOL_fname, AUGM_FINE_TUNING_MAXIMUM_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( AUGM_FINE_TUNING_MAXIMUM_FUSION_FLATTEN_fname, AUGM_FINE_TUNING_MAXIMUM_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( AUGM_FINE_TUNING_MAXIMUM_FUSION_MAX_POOL_fname, AUGM_FINE_TUNING_MAXIMUM_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) else: if fusion_strategy == 'concatenate': if pooling_mode == 'avg': weights_path = get_file( FINE_TUNING_CONCATENATE_FUSION_AVG_POOL_fname, FINE_TUNING_CONCATENATE_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( FINE_TUNING_CONCATENATE_FUSION_FLATTEN_fname, FINE_TUNING_CONCATENATE_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( FINE_TUNING_CONCATENATE_FUSION_MAX_POOL_fname, FINE_TUNING_CONCATENATE_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif fusion_strategy == 'average': if pooling_mode == 'avg': weights_path = get_file( FINE_TUNING_AVERAGE_FUSION_AVG_POOL_fname, FINE_TUNING_AVERAGE_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( FINE_TUNING_AVERAGE_FUSION_FLATTEN_fname, FINE_TUNING_AVERAGE_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( FINE_TUNING_AVERAGE_FUSION_MAX_POOL_fname, FINE_TUNING_AVERAGE_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif fusion_strategy == 'maximum': if pooling_mode == 'avg': weights_path = get_file( FINE_TUNING_MAXIMUM_FUSION_AVG_POOL_fname, FINE_TUNING_MAXIMUM_FUSION_AVG_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'flatten': weights_path = get_file( FINE_TUNING_MAXIMUM_FUSION_FLATTEN_fname, FINE_TUNING_MAXIMUM_FUSION_FLATTEN_WEIGHTS_PATH, cache_subdir=cache_subdir) elif pooling_mode == 'max': weights_path = get_file( FINE_TUNING_MAXIMUM_FUSION_MAX_POOL_fname, FINE_TUNING_MAXIMUM_FUSION_MAX_POOL_WEIGHTS_PATH, cache_subdir=cache_subdir) else: if fusion_strategy == 'average': weights_path = get_file( FINE_TUNING_AVERAGE_FUSION_NO_TOP_fname, FINE_TUNING_AVERAGE_FUSION_WEIGHTS_PATH_NO_TOP, cache_subdir=cache_subdir) elif fusion_strategy == 'concatenate': weights_path = get_file( FINE_TUNING_CONCATENATE_FUSION_NO_TOP_fname, FINE_TUNING_CONCATENATE_FUSION_WEIGHTS_PATH_NO_TOP, cache_subdir=cache_subdir) elif fusion_strategy == 'maximum': weights_path = get_file( FINE_TUNING_MAXIMUM_FUSION_NO_TOP_fname, FINE_TUNING_MAXIMUM_FUSION_WEIGHTS_PATH_NO_TOP, cache_subdir=cache_subdir) model.load_weights(weights_path) return model
def compoundNet_feature_extraction(object_centric_model='VGG16', scene_centric_model='VGG16_Places365', fusion_strategy='concatenate', pooling_mode='avg', classes=9, data_augm_enabled=False): """ConvNet as fixed feature extractor, consist of taking the convolutional base of a previously-trained network, running the new data through it, and training a new classifier on top of the output. (i.e. train only the randomly initialized top layers while freezing all convolutional layers of the original model). # Arguments object_centric_model: one of `VGG16`, `VGG19` or `ResNet50` scene_centric_model: `VGG16_Places365` fusion_strategy: one of `concatenate` (feature vectors of different sources are concatenated into one super-vector), `average` (the feature set is averaged) or `maximum` (selects the highest value from the corresponding features). pooling_mode: Optional pooling_mode mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling_mode will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling_mode will be applied. classes: optional number of classes to classify images into, only to be specified if `weights` argument is `None`. data_augm_enabled: whether to use the augmented samples during training. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `object_centric_model`, `pooling_mode`, `fusion_strategy` , `scene_centric_model` or invalid input shape. """ if not (object_centric_model in {'VGG16', 'VGG19', 'ResNet50'}): raise ValueError( 'The `scene_centric_model` argument should be either ' '`VGG16`, `VGG19` or `ResNet50`. Other models will be supported in future releases. ' ) if not (pooling_mode in {'avg', 'max', 'flatten'}): raise ValueError('The `pooling_mode` argument should be either ' '`avg` (GlobalAveragePooling2D), `max` ' '(GlobalMaxPooling2D), ' 'or `flatten` (Flatten).') if not (fusion_strategy in {'concatenate', 'average', 'maximum'}): raise ValueError( 'The `fusion_strategy` argument should be either ' '`concatenate` (feature vectors of different sources are concatenated into one super-vector),' ' `average` (the feature set is averaged) ' 'or `maximum` (selects the highest value from the corresponding features).' ) if not (scene_centric_model in {'VGG16_Places365'}): raise ValueError( 'The `scene_centric_model` argument should be ' '`VGG16_Places365`. Other models will be supported in future releases.' ) # Define the name of the model and its weights weights_name = 'compoundNet_feature_extraction_' \ + object_centric_model + '_' \ + fusion_strategy + '_fusion_' \ + pooling_mode + '_pool_weights_tf_dim_ordering_tf_kernels.h5' augm_samples_weights_name = 'augm_compoundNet_feature_extraction_' \ + object_centric_model + '_' \ + fusion_strategy + '_fusion_' \ + pooling_mode + '_pool_weights_tf_dim_ordering_tf_kernels.h5' model_log = logs_dir + 'compoundNet_feature_extraction_' \ + object_centric_model + '_' \ + fusion_strategy + '_fusion_' \ + pooling_mode + '_pool_log.csv' csv_logger = CSVLogger(model_log, append=True, separator=',') input_tensor = Input(shape=(224, 224, 3)) # create the base object_centric_model pre-trained model for warm-up if object_centric_model == 'VGG16': object_base_model = VGG16(input_tensor=input_tensor, weights='imagenet', include_top=False) elif object_centric_model == 'VGG19': object_base_model = VGG19(input_tensor=input_tensor, weights='imagenet', include_top=False) elif object_centric_model == 'ResNet50': tmp_model = ResNet50(input_tensor=input_tensor, weights='imagenet', include_top=False) object_base_model = Model( inputs=tmp_model.input, outputs=tmp_model.get_layer('activation_48').output) print('\n \n') print('The plain, object-centric `' + object_centric_model + '` pre-trained convnet was successfully initialised.\n') scene_base_model = VGG16_Places365(input_tensor=input_tensor, weights='places', include_top=False) print('The plain, scene-centric `' + scene_centric_model + '` pre-trained convnet was successfully initialised.\n') # retrieve the ouputs object_base_model_output = object_base_model.output scene_base_model_output = scene_base_model.output # We will feed the extracted features to a merging layer if fusion_strategy == 'concatenate': merged = concatenate( [object_base_model_output, scene_base_model_output]) elif fusion_strategy == 'average': merged = average([object_base_model_output, scene_base_model_output]) else: merged = maximum([object_base_model_output, scene_base_model_output]) if pooling_mode == 'avg': x = GlobalAveragePooling2D(name='GAP')(merged) elif pooling_mode == 'max': x = GlobalMaxPooling2D(name='GMP')(merged) elif pooling_mode == 'flatten': x = Flatten(name='FLATTEN')(merged) x = Dense(256, activation='relu', name='FC1')(x) # let's add a fully-connected layer # When random init is enabled, we want to include Dropout, # otherwise when loading a pre-trained HRA model we want to omit # Dropout layer so the visualisations are done properly (there is an issue if it is included) x = Dropout(0.5, name='DROPOUT')(x) # and a logistic layer with the number of classes defined by the `classes` argument predictions = Dense(classes, activation='softmax', name='PREDICTIONS')(x) # new softmax layer # this is the transfer learning model we will train model = Model(inputs=object_base_model.input, outputs=predictions) print( 'Randomly initialised classifier was successfully added on top of the merged outputs. \n' ) print( 'Number of trainable weights before freezing the conv. bases of the respective original models: ' '' + str(len(model.trainable_weights))) # first: train only the top layers (which were randomly initialized) # i.e. freeze all convolutional layers of the preliminary base model for layer in object_base_model.layers: layer.trainable = False for layer in scene_base_model.layers: layer.trainable = False print( 'Number of trainable weights after freezing the conv. bases of the respective original models: ' '' + str(len(model.trainable_weights))) print('\n') # compile the warm_up_model (should be done *after* setting layers to non-trainable) model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy']) model.summary() # # The attribute model.metrics_names will give you the display labels for the scalar outputs. # print warm_up_model.metrics_names if data_augm_enabled: print( 'Using augmented samples for training. This may take a while ! \n') t = now() history = model.fit_generator(augmented_train_generator, steps_per_epoch=nb_train_samples // batch_size, epochs=feature_extraction_epochs, callbacks=[csv_logger], class_weight=class_weight) print( 'Training time for re-training the last Dense layer using augmented samples: %s' % (now() - t)) model.save_weights(feature_extraction_dir + augm_samples_weights_name) print('Model weights using augmented samples were saved as `' + augm_samples_weights_name + '`') print('\n') else: t = now() history = model.fit_generator(train_generator, steps_per_epoch=nb_train_samples // batch_size, epochs=feature_extraction_epochs, callbacks=[csv_logger], class_weight=class_weight) print('Training time for re-training the last Dense layer: %s' % (now() - t)) model.save_weights(feature_extraction_dir + weights_name) print('Model weights were saved as `' + weights_name + '`') print('\n') return model
def MI_Net_with_DS(dataset): """Train and evaluate on MI-Net with deep supervision. Parameters ----------------- dataset : dict A dictionary contains all dataset information. We split train/test by keys. Returns ----------------- test_acc : float Testing accuracy of MI-Net with deep supervision. """ # load data and convert type train_bags = dataset['train'] test_bags = dataset['test'] # convert bag to batch train_set = convertToBatch(train_bags) test_set = convertToBatch(test_bags) dimension = train_set[0][0].shape[1] weight = [1.0, 1.0, 1.0, 0.0] # data: instance feature, n*d, n = number of training instance data_input = Input(shape=(dimension, ), dtype='float32', name='input') # fully-connected fc1 = Dense(256, activation='relu', kernel_regularizer=l2(args.weight_decay))(data_input) fc2 = Dense(128, activation='relu', kernel_regularizer=l2(args.weight_decay))(fc1) fc3 = Dense(64, activation='relu', kernel_regularizer=l2(args.weight_decay))(fc2) # dropout dropout1 = Dropout(rate=0.5)(fc1) dropout2 = Dropout(rate=0.5)(fc2) dropout3 = Dropout(rate=0.5)(fc3) # features pooling fp1 = Feature_pooling(output_dim=1, kernel_regularizer=l2(args.weight_decay), pooling_mode=args.pooling_mode, name='fp1')(dropout1) fp2 = Feature_pooling(output_dim=1, kernel_regularizer=l2(args.weight_decay), pooling_mode=args.pooling_mode, name='fp2')(dropout2) fp3 = Feature_pooling(output_dim=1, kernel_regularizer=l2(args.weight_decay), pooling_mode=args.pooling_mode, name='fp3')(dropout3) # score average mg_ave = average([fp1, fp2, fp3], name='ave') model = Model(inputs=[data_input], outputs=[fp1, fp2, fp3, mg_ave]) sgd = SGD(lr=args.init_lr, decay=1e-4, momentum=args.momentum, nesterov=True) model.compile(loss={ 'fp1': bag_loss, 'fp2': bag_loss, 'fp3': bag_loss, 'ave': bag_loss }, loss_weights={ 'fp1': weight[0], 'fp2': weight[1], 'fp3': weight[2], 'ave': weight[3] }, optimizer=sgd, metrics=[bag_accuracy]) # train model t1 = time.time() num_batch = len(train_set) for epoch in range(args.max_epoch): train_loss, train_acc = train_eval(model, train_set) test_loss, test_acc = test_eval(model, test_set) print 'epoch=', epoch, ' train_loss= {:.3f}'.format( train_loss), ' train_acc= {:.3f}'.format( train_acc), ' test_loss={:.3f}'.format( test_loss), ' test_acc= {:.3f}'.format(test_acc) t2 = time.time() print 'run time:', (t2 - t1) / 60, 'min' print 'test_acc={:.3f}'.format(test_acc) return test_acc
def getModel(training): inputShape = (128, 32, 1) kernelVals = [5, 5, 3, 3, 3] convFilters = [32, 64, 128, 128, 256] strideVals = [(2, 2), (2, 2), (1, 2), (1, 2), (1, 2)] rnnUnits = 256 maxStringLen = 32 inputs = Input(name='inputX', shape=inputShape, dtype='float32') inner = inputs for i in range(len(kernelVals)): inner = Conv2D(convFilters[i],(kernelVals[i],kernelVals[i]),padding = 'same',\ name = 'conv'+str(i), kernel_initializer = 'he_normal')(inner) inner = BatchNormalization()(inner) inner = Activation('relu')(inner) inner = MaxPooling2D(pool_size=strideVals[i], name='max' + str(i + 1))(inner) inner = Reshape(target_shape=(maxStringLen, rnnUnits), name='reshape')(inner) LSF = LSTM(rnnUnits, return_sequences=True, kernel_initializer='he_normal', name='LSTM1F')(inner) LSB = LSTM(rnnUnits, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='LSTM1B')(inner) LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB) LS1 = average([LSF, LSB]) LS1 = BatchNormalization()(LS1) LSF = LSTM(rnnUnits, return_sequences=True, kernel_initializer='he_normal', name='LSTM2F')(LS1) LSB = LSTM(rnnUnits, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='LSTM2B')(LS1) LSB = Lambda(lambda inputTensor: K.reverse(inputTensor, axes=1))(LSB) LS2 = concatenate([LSF, LSB]) LS2 = BatchNormalization()(LS2) yPred = Dense(len(unicodes) + 1, kernel_initializer='he_normal', name='dense2')(LS2) yPred = Activation('softmax', name='softmax')(yPred) #Model(inputs = inputs,outputs = yPred).summary() labels = Input(name='label', shape=[32], dtype='float32') inputLength = Input(name='inputLen', shape=[1], dtype='int64') # (None, 1) labelLength = Input(name='labelLen', shape=[1], dtype='int64') lossOut = Lambda(ctcLambdaFunc, output_shape=(1, ), name='ctc')([yPred, labels, inputLength, labelLength]) if training: return Model(inputs=[inputs, labels, inputLength, labelLength], outputs=[lossOut, yPred]) return Model(inputs=[inputs], outputs=yPred)
def get_unet_512(input_shape=(512, 512, 3), num_classes=1): inputs = Input(shape=input_shape) # 512 down0 = block(inputs, 16) down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0) # 256 down1 = block(down0_pool, 32) down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1) # 128 down2 = block(down1_pool, 64) down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2) # 64 down3 = block(down2_pool, 128) down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3) # 32 down4 = block(down3_pool, 256) down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4) # 16 down5 = block(down4_pool, 512) down5_pool = MaxPooling2D((2, 2), strides=(2, 2))(down5) # 8 center = block(down5_pool, 1024) # stacked dilated convolution # dilate1 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=1)(down5_pool) # dilate2 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=2)(dilate1) # dilate3 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=4)(dilate2) # dilate4 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=8)(dilate3) # dilate5 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=16)(dilate4) # dilate6 = Conv2D(1024, (3, 3), activation='relu', padding='same', dilation_rate=32)(dilate5) # center = add([dilate1, dilate2, dilate3, dilate4, dilate5, dilate6]) # center up5 = UpSampling2D((2, 2))(center) up5 = concatenate([down5, up5], axis=3) up5 = block(up5, 512) # 16 up4 = UpSampling2D((2, 2))(up5) up4 = concatenate([down4, up4], axis=3) up4 = block(up4, 256) # 32 up3 = UpSampling2D((2, 2))(up4) up3 = concatenate([down3, up3], axis=3) up3 = block(up3, 128) # 64 up2 = UpSampling2D((2, 2))(up3) up2 = concatenate([down2, up2], axis=3) up2 = block(up2, 64) # 128 up1 = UpSampling2D((2, 2))(up2) up1 = concatenate([down1, up1], axis=3) up1 = block(up1, 32) # 256 up0 = UpSampling2D((2, 2))(up1) up0 = concatenate([down0, up0], axis=3) up0 = block(up0, 16) # 512 output1 = Conv2D(num_classes, (1, 1), activation='sigmoid')(up0) if not USE_REFINE_NET: model = Model(inputs=inputs, outputs=[output1]) # model.load_weights('../weights/head-segmentation-model.h5') # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss='binary_crossentropy', metrics=[dice_loss]) return model else: inputs2 = concatenate([inputs, output1]) outputs2 = block2(inputs2, 64) outputs2 = average([output1, outputs2]) model2 = Model(inputs=inputs, outputs=[output1, outputs2]) return model2
def save_basic_ensemble_evals(sequences, test_data, scratch_model_func, weight_dirname, eval_dirname, force=False): # Get test data x_test, y_test = test_data nb_classes = y_test.shape[1] # Iterate through all provided Snapshot models for cur_seq, init_lrs, iter_epochs, restart_indices in sequences: base_string = snap_train.get_checkpoint_base_string( cur_seq, init_lrs, iter_epochs, restart_indices) max_size = len(cur_seq) if max_size < 2: continue # Get biggest sized ensemble and ensemble omitting first model for cur_size in range(max_size - 1, max_size + 1): eval_filename = eval_dirname + base_string + (".%d.eval" % cur_size) if not os.path.isfile(eval_filename) or force: # Get checkpoint weights tmp_chkpt_names = [] first_checkpt = base_string + ".0.hdf5" for filename in os.listdir(weight_dirname): if filename.startswith(base_string): tmp_chkpt_names.append(weight_dirname + filename) # Check to ensure that correct number of files were found chkpt_names = trim_snap_weight_list(tmp_chkpt_names, base_string, cur_size) if not chkpt_names: continue # Get individual models and their averages print("\n\nGetting averages of individual Snapshot models... ") running_vals = [0.0, 0.0, 0.0] chkpt_models = [] for i, name in enumerate(chkpt_names): model = scratch_model_func() model.load_weights(name) for j, cur_layer in enumerate(model.layers): cur_layer.name = ("%s_%d_%d" % (base_string, i, j)) # Keep track of model averages model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[ 'accuracy', metrics.top_k_categorical_accuracy ]) for j, val in enumerate( model.evaluate(x_test, y_test, batch_size=64, verbose=0)): running_vals[j] += (float(val) / cur_size) # Add checkpoint model to list of models chkpt_models.append(model) # Create Snapshot Ensemble model chkpt_inputs = [model.input for model in chkpt_models] chkpt_outputs = [model.output for model in chkpt_models] # Usage of "average" vs. "keras.layers.merge.Average": # https://github.com/fchollet/keras/issues/3921 final_output = average(chkpt_outputs) snapshot_model = Model(inputs=chkpt_inputs, outputs=final_output) snapshot_model.compile( optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', metrics.top_k_categorical_accuracy]) # Save eval metrics (in eval_dirname) print("\n\nEvaluating the following Snapshot Ensemble: %s" % eval_filename) eval_metrics = snapshot_model.evaluate([x_test] * cur_size, y_test, batch_size=12, verbose=1) print(snapshot_model.metrics_names) print(eval_metrics) running_vals.extend(eval_metrics) pickle.dump(running_vals, open(eval_filename, "wb")) # Return nothing return
def miccai2018_net(vol_size, enc_nf, dec_nf, int_steps=7, use_miccai_int=False, indexing='ij', bidir=False, vel_resize=1/2): """ architecture for probabilistic diffeomoprhic VoxelMorph presented in the MICCAI 2018 paper. You may need to modify this code (e.g., number of layers) to suit your project needs. The stationary velocity field operates in a space (0.5)^3 of vol_size for computational reasons. :param vol_size: volume size. e.g. (256, 256, 256) :param enc_nf: list of encoder filters. right now it needs to be 1x4. e.g. [16,32,32,32] :param dec_nf: list of decoder filters. right now it must be 1x6, see unet function. :param use_miccai_int: whether to use the manual miccai implementation of scaling and squaring integration note that the 'velocity' field outputted in that case was since then we've updated the code to be part of a flexible layer. see neuron.layers.VecInt **This param will be phased out (set to False behavior)** :param int_steps: the number of integration steps :param indexing: xy or ij indexing. we recommend ij indexing if training from scratch. miccai 2018 runs were done with xy indexing. **This param will be phased out (set to 'ij' behavior)** :return: the keras model """ ndims = len(vol_size) assert ndims in [1, 2, 3], "ndims should be one of 1, 2, or 3. found: %d" % ndims # get unet unet_model = unet_model_3d(vol_size, batch_normalization=True) [src, tgt] = unet_model.inputs x_out, up0, up1, up2, flow_params0, flow_params1, flow_params2, y0, y1, y2 = unet_model.outputs # print("unet input", src.shape) # print("unet output", x_out.shape) # velocity mean and logsigma layers Conv = getattr(KL, 'Conv%dD' % ndims) flow_mean = Conv(ndims, kernel_size=3, padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow', data_format="channels_last")(x_out) # we're going to initialize the velocity variance very low, to start stable. flow_log_sigma = Conv(ndims, kernel_size=3, padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-10), bias_initializer=keras.initializers.Constant(value=-10), name='log_sigma', data_format="channels_last")(x_out) flow_params = concatenate([flow_mean, flow_log_sigma]) # velocity sample flow = Sample(name="z_sample")([flow_mean, flow_log_sigma]) from keras.layers.merge import average flow = average([flow, up0, up1, up2]) # integrate if diffeomorphic (i.e. treating 'flow' above as stationary velocity field) if use_miccai_int: v = flow for _ in range(int_steps): v1 = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([v, v]) v = keras.layers.add([v, v1]) flow = v else: # new implementation in neuron is cleaner. z_sample = flow flow = nrn_layers.VecInt(method='ss', name='flow-int', int_steps=int_steps)(z_sample) if bidir: rev_z_sample = Negate()(z_sample) neg_flow = nrn_layers.VecInt(method='ss', name='neg_flow-int', int_steps=int_steps)(rev_z_sample) # get up to final resolution flow = trf_resize(flow, vel_resize, name='diffflow') if bidir: neg_flow = trf_resize(neg_flow, vel_resize, name='neg_diffflow') # transform y = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([src, flow]) if bidir: y_tgt = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([tgt, neg_flow]) # prepare outputs and losses outputs = [y, flow_params, flow_params0, flow_params1, flow_params2, y0, y1, y2] if bidir: outputs = [y, y_tgt, flow_params] # build the model return Model(inputs=[src, tgt], outputs=outputs)
outs = [] #the list of ensemble outputs for i in range(ens_models): # Conv[32] -> Conv[32] -> Pool (with dropout on the pooling layer), applying BN in between conv1 = Convolution2D(conv_depth, kernel_size, kernel_size, border_mode = 'same', init = 'he_uniform', W_regularizer = l2(l2_lambda), activation = 'relu')(inp_norm) conv1 = BatchNormalization(axis = 1)(conv1) conv2 = Convolution2D(conv_depth, kernel_size, kernel_size, border_mode = 'same', init = 'he_uniform', W_regularizer = l2(l2_lambda), activation = 'relu')(conv1) conv2 = BatchNormalization(axis = 1)(conv2) pool_1 = MaxPooling2D(pool_size = (pool_size, pool_size), dim_ordering="th")(conv2) drop_1 = Dropout(drop_prob_1)(pool_1) flat = Flatten()(drop_1) hidden = Dense(hidden_size, init = 'he_uniform', W_regularizer = l2(l2_lambda), activation = 'relu')(flat) # Hidden ReLU layer hidden = BatchNormalization(axis = 1)(hidden) drop = Dropout(drop_prob_2)(hidden) outs.append(Dense(num_classes, init = 'glorot_uniform', W_regularizer = l2(l2_lambda), activation = 'softmax')(drop)) # Output softmax layer out = average(outs) # Avarage the predictioins to obtain the final output model = Model(input = inp, output = out) model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy']) datagen = ImageDataGenerator(width_shift_range = 0.1, height_shift_range = 0.1) # randomly shift images horizontally and vertically datagen.fit(X_train) model.fit_generator(datagen.flow(X_train, Y_train, batch_size = batch_size), samples_per_epoch = X_train.shape[0], nb_epoch = num_epochs, validation_data = (X_val, Y_val), verbose = 1, callbacks = [EarlyStopping(monitor = 'val_loss', patience = 5)]) res = model.evaluate(X_test, Y_test, verbose = 1) print("Res = ", res)
def get_unet_128(input_shape=(128, 128, 3), num_classes=1): inputs = Input(shape=input_shape) # 128 down1 = Conv2D(16, (3, 3), padding='same')(inputs) down1 = BatchNormalization()(down1) down1 = Activation('relu')(down1) down1 = Conv2D(16, (3, 3), padding='same')(down1) down1 = BatchNormalization()(down1) down1 = Activation('relu')(down1) down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1) # 64 down2 = Conv2D(32, (3, 3), padding='same')(down1_pool) down2 = BatchNormalization()(down2) down2 = Activation('relu')(down2) down2 = Conv2D(32, (3, 3), padding='same')(down2) down2 = BatchNormalization()(down2) down2 = Activation('relu')(down2) down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2) # 32 down3 = Conv2D(64, (3, 3), padding='same')(down2_pool) down3 = BatchNormalization()(down3) down3 = Activation('relu')(down3) down3 = Conv2D(64, (3, 3), padding='same')(down3) down3 = BatchNormalization()(down3) down3 = Activation('relu')(down3) down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3) # 16 down4 = Conv2D(128, (3, 3), padding='same')(down3_pool) down4 = BatchNormalization()(down4) down4 = Activation('relu')(down4) down4 = Conv2D(128, (3, 3), padding='same')(down4) down4 = BatchNormalization()(down4) down4 = Activation('relu')(down4) down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4) # 8 center = Conv2D(256, (3, 3), padding='same')(down4_pool) center = BatchNormalization()(center) center = Activation('relu')(center) center = Conv2D(256, (3, 3), padding='same')(center) center = BatchNormalization()(center) center = Activation('relu')(center) # center up4 = UpSampling2D((2, 2))(center) up4 = concatenate([down4, up4], axis=3) up4 = Conv2D(128, (3, 3), padding='same')(up4) up4 = BatchNormalization()(up4) up4 = Activation('relu')(up4) up4 = Conv2D(128, (3, 3), padding='same')(up4) up4 = BatchNormalization()(up4) up4 = Activation('relu')(up4) up4 = Conv2D(128, (3, 3), padding='same')(up4) up4 = BatchNormalization()(up4) up4 = Activation('relu')(up4) # 16 up3 = UpSampling2D((2, 2))(up4) up3 = concatenate([down3, up3], axis=3) up3 = Conv2D(64, (3, 3), padding='same')(up3) up3 = BatchNormalization()(up3) up3 = Activation('relu')(up3) up3 = Conv2D(64, (3, 3), padding='same')(up3) up3 = BatchNormalization()(up3) up3 = Activation('relu')(up3) up3 = Conv2D(64, (3, 3), padding='same')(up3) up3 = BatchNormalization()(up3) up3 = Activation('relu')(up3) # 32 up2 = UpSampling2D((2, 2))(up3) up2 = concatenate([down2, up2], axis=3) up2 = Conv2D(32, (3, 3), padding='same')(up2) up2 = BatchNormalization()(up2) up2 = Activation('relu')(up2) up2 = Conv2D(32, (3, 3), padding='same')(up2) up2 = BatchNormalization()(up2) up2 = Activation('relu')(up2) up2 = Conv2D(32, (3, 3), padding='same')(up2) up2 = BatchNormalization()(up2) up2 = Activation('relu')(up2) # 64 up1 = UpSampling2D((2, 2))(up2) up1 = concatenate([down1, up1], axis=3) up1 = Conv2D(16, (3, 3), padding='same')(up1) up1 = BatchNormalization()(up1) up1 = Activation('relu')(up1) up1 = Conv2D(16, (3, 3), padding='same')(up1) up1 = BatchNormalization()(up1) up1 = Activation('relu')(up1) up1 = Conv2D(16, (3, 3), padding='same')(up1) up1 = BatchNormalization()(up1) up1 = Activation('relu')(up1) # 128 output1 = Conv2D(num_classes, (1, 1), activation='sigmoid')(up1) if not USE_REFINE_NET: model = Model(inputs=inputs, outputs=[output1]) # model.load_weights('../weights/head-segmentation-model.h5') # model.compile(optimizer=SGD(lr=0.01, momentum=0.9), loss='binary_crossentropy', metrics=[dice_loss]) return model else: inputs2 = concatenate([inputs, output1]) outputs2 = block2(inputs2, 64) outputs2 = average([output1, outputs2]) model2 = Model(inputs=inputs, outputs=[output1, outputs2]) return model2 return model
def Model3_LSTM_BiLSTM_LSTM(wordvocabsize, targetvocabsize, charvobsize, word_W, char_W, input_fragment_lenth, input_leftcontext_lenth, input_rightcontext_lenth, input_maxword_length, w2v_k, c2v_k, hidden_dim=200, batch_size=32, optimizer='rmsprop'): hidden_dim = 100 word_input_fragment = Input(shape=(input_fragment_lenth, ), dtype='int32') word_embedding_fragment = Embedding(input_dim=wordvocabsize + 1, output_dim=w2v_k, input_length=input_fragment_lenth, mask_zero=False, trainable=True, weights=[word_W])(word_input_fragment) word_embedding_fragment = Dropout(0.5)(word_embedding_fragment) char_input_fragment = Input(shape=( input_fragment_lenth, input_maxword_length, ), dtype='int32') char_embedding_fragment = TimeDistributed( Embedding(input_dim=charvobsize, output_dim=c2v_k, batch_input_shape=(batch_size, input_fragment_lenth, input_maxword_length), mask_zero=False, trainable=True, weights=[char_W]))(char_input_fragment) char_cnn_fragment = TimeDistributed( Conv1D(50, 3, activation='relu', padding='valid')) char_embedding_fragment = char_cnn_fragment(char_embedding_fragment) char_embedding_fragment = TimeDistributed( GlobalMaxPooling1D())(char_embedding_fragment) char_embedding_fragment = Dropout(0.25)(char_embedding_fragment) word_input_leftcontext = Input(shape=(input_leftcontext_lenth, ), dtype='int32') word_embedding_leftcontext = Embedding( input_dim=wordvocabsize + 1, output_dim=w2v_k, input_length=input_leftcontext_lenth, mask_zero=True, trainable=True, weights=[word_W])(word_input_leftcontext) word_embedding_leftcontext = Dropout(0.5)(word_embedding_leftcontext) char_input_leftcontext = Input(shape=( input_leftcontext_lenth, input_maxword_length, ), dtype='int32') char_input_rightcontext = Input(shape=( input_rightcontext_lenth, input_maxword_length, ), dtype='int32') word_input_rightcontext = Input(shape=(input_rightcontext_lenth, ), dtype='int32') word_embedding_rightcontext = Embedding( input_dim=wordvocabsize + 1, output_dim=w2v_k, input_length=input_rightcontext_lenth, mask_zero=True, trainable=True, weights=[word_W])(word_input_rightcontext) word_embedding_rightcontext = Dropout(0.5)(word_embedding_rightcontext) embedding_fragment = concatenate( [word_embedding_fragment, char_embedding_fragment], axis=-1) embedding_leftcontext = word_embedding_leftcontext embedding_rightcontext = word_embedding_rightcontext LSTM_leftcontext = LSTM(hidden_dim, go_backwards=False, activation='tanh')(embedding_leftcontext) Rep_LSTM_leftcontext = RepeatVector(input_fragment_lenth)(LSTM_leftcontext) LSTM_rightcontext = LSTM(hidden_dim, go_backwards=True, activation='tanh')(embedding_rightcontext) Rep_LSTM_rightcontext = RepeatVector(input_fragment_lenth)( LSTM_rightcontext) BiLSTM_fragment = Bidirectional(LSTM(hidden_dim // 2, activation='tanh', return_sequences=True), merge_mode='concat')(embedding_fragment) context_ADD = add([LSTM_leftcontext, BiLSTM_fragment, LSTM_rightcontext]) context_subtract_l = subtract([BiLSTM_fragment, LSTM_leftcontext]) context_subtract_r = subtract([BiLSTM_fragment, LSTM_rightcontext]) context_average = average( [LSTM_leftcontext, BiLSTM_fragment, LSTM_rightcontext]) context_maximum = maximum( [LSTM_leftcontext, BiLSTM_fragment, LSTM_rightcontext]) embedding_mix = concatenate([ embedding_fragment, BiLSTM_fragment, context_ADD, context_subtract_l, context_subtract_r, context_average, context_maximum ], axis=-1) # BiLSTM_fragment = Bidirectional(LSTM(hidden_dim // 2, activation='tanh'), merge_mode='concat')(embedding_fragment) decoderlayer1 = Conv1D(50, 1, activation='relu', strides=1, padding='same')(embedding_mix) decoderlayer2 = Conv1D(50, 2, activation='relu', strides=1, padding='same')(embedding_mix) decoderlayer3 = Conv1D(50, 3, activation='relu', strides=1, padding='same')(embedding_mix) decoderlayer4 = Conv1D(50, 4, activation='relu', strides=1, padding='same')(embedding_mix) CNNs_fragment = concatenate( [decoderlayer1, decoderlayer2, decoderlayer3, decoderlayer4], axis=-1) CNNs_fragment = Dropout(0.5)(CNNs_fragment) CNNs_fragment = GlobalMaxPooling1D()(CNNs_fragment) concat = Dropout(0.3)(CNNs_fragment) output = Dense(targetvocabsize, activation='softmax')(concat) Models = Model([ word_input_fragment, word_input_leftcontext, word_input_rightcontext, char_input_fragment, char_input_leftcontext, char_input_rightcontext ], output) Models.compile(loss='categorical_crossentropy', optimizer=optimizers.RMSprop(lr=0.001), metrics=['acc']) return Models