def mean_squared_error(inputs): if len(inputs) == 2: [mu1, mu2] = inputs if isinstance(mu2, list): return average([mse(mu1, pred) for pred in mu2]) else: return mse(mu1, mu2) else: true = inputs[0] return average( [mse(true, inputs[pred]) for pred in range(1, len(inputs))])
def categorical_crossentropy(inputs): print("Categorical cross entropy inputs : ", inputs) if len(inputs) == 2: [mu1, mu2] = inputs if isinstance(mu2, list): return average( [K.categorical_crossentropy(mu1, pred) for pred in mu2]) else: return K.categorical_crossentropy(mu1, mu2) else: true = inputs[0] return average([ K.categorical_crossentropy(true, inputs[pred]) for pred in range(1, len(inputs)) ])
def binary_crossentropy(inputs): if len(inputs) == 2: [mu1, mu2] = inputs if isinstance(mu2, list): return average([K.binary_crossentropy(mu1, pred) for pred in mu2]) else: return K.binary_crossentropy(mu1, mu2) else: raise Exception( "BINARY CROSS ENTROPY HAS MORE THAN 2 ARGUMENTS. ENSURE THIS IS DESIRED" ) true = inputs[0] return average([ K.binary_crossentropy(true, inputs[pred]) for pred in range(1, len(inputs)) ])
def _embedder(inputs): assert len(inputs) == len(input_names), \ f'Expected {len(input_names)} input(s) {input_names}, ' \ f'but got {len(inputs)}' x = [] for n, inp in zip(input_names, inputs): for layer in embedder_layers[n]: inp = layer(inp) x += [inp] if len(x) > 1: if merge_mode == 'concat': x = layers.concatenate(x) elif merge_mode == 'sum': x = layers.add(x) elif merge_mode == 'ave': x = layers.average(x) else: raise ValueError( 'Unrecognized merge mode {}'.format(merge_mode)) else: x = x[0] if project_dim is not None: x = layers.Dense(project_dim, input_shape=(emb_dim, ))(x) return layers.Dropout(rate=dropout)(x)
def DeepModel(size_set=640): img_input = Input(shape=(size_set, size_set, 3)) conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input) conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', name='block1_conv2')(conv1) pool1 = MaxPool2D(pool_size=(2, 2))(conv1) conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block2_conv1')(pool1) conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block2_conv2')(conv2) pool2 = MaxPool2D(pool_size=(2, 2))(conv2) conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block3_conv1')(pool2) conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block3_conv2')(conv3) pool3 = MaxPool2D(pool_size=(2, 2))(conv3) conv4 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block4_conv1')(pool3) conv4 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block4_conv2')(conv4) pool4 = MaxPool2D(pool_size=(2, 2))(conv4) conv5 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(pool4) conv5 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(conv5) up6 = concatenate( [Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same', name='block6_dconv')(conv5), conv4], axis=3) conv6 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block6_conv1')(up6) conv6 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block6_conv2')(conv6) up7 = concatenate( [Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same', name='block7_dconv')(conv6), conv3], axis=3) conv7 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block7_conv1')(up7) conv7 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block7_conv2')(conv7) up8 = concatenate([Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same', name='block8_dconv')(conv7), conv2], axis=3) conv8 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block8_conv1')(up8) conv8 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block8_conv2')(conv8) up9 = concatenate([Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same', name='block9_dconv')(conv8), conv1], axis=3) conv9 = Conv2D(32, (3, 3), activation='relu', padding='same', name='block9_conv1')(up9) conv9 = Conv2D(32, (3, 3), activation='relu', padding='same', name='block9_conv2')(conv9) side6 = UpSampling2D(size=(8, 8))(conv6) side7 = UpSampling2D(size=(4, 4))(conv7) side8 = UpSampling2D(size=(2, 2))(conv8) out6 = Conv2D(1, (1, 1), activation='sigmoid', name='side_6')(side6) out7 = Conv2D(1, (1, 1), activation='sigmoid', name='side_7')(side7) out8 = Conv2D(1, (1, 1), activation='sigmoid', name='side_8')(side8) out9 = Conv2D(1, (1, 1), activation='sigmoid', name='side_9')(conv9) out10 = average([out6, out7, out8, out9]) return Model(inputs=img_input, outputs=out10)
def main(): model1 = get_model() model2 = get_model() model3 = get_model() inputs = keras.Input(shape=(128,)) y1 = model1(inputs) y2 = model2(inputs) y3 = model3(inputs) outputs = layers.average([y1, y2, y3]) ensemble_model = keras.Model(inputs=inputs, outputs=outputs, name="function_define_model") ensemble_model.summary()
def two_stream_fuse(self): # spatial stream (frozen) cnn_spatial_multi = self.cnn_spatial_multi() # temporal stream (frozen) cnn_temporal_multi = self.cnn_temporal_multi() # fused by taking average outputs = average([cnn_spatial_multi.output, cnn_temporal_multi.output]) model = Model([cnn_spatial_multi.input, cnn_temporal_multi.input], outputs) return model
def __new__(cls, mode: str) \ -> KM.Model: assert mode in ['base', 'skip', 'bn'] inputs = KL.Input(shape=[None, None, 3], name="input_image") x = inputs x = KL.Conv2D(64, (3, 3), padding="SAME", kernel_initializer='random_uniform', bias_initializer='zeros', name="layer1")(x) if mode == 'bn': x = KL.BatchNormalization()(x) x = KL.ReLU()(x) x = KL.Conv2D(64, (3, 3), padding="SAME", kernel_initializer='random_uniform', bias_initializer='zeros', name="layer2")(x) if mode == 'bn': x = KL.BatchNormalization()(x) x = KL.ReLU()(x) x = KL.Conv2D(64, (3, 3), padding="SAME", kernel_initializer='random_uniform', bias_initializer='zeros', name="layer3")(x) if mode == 'bn': x = KL.BatchNormalization()(x) x = KL.ReLU()(x) x = KL.Conv2D(64, (3, 3), padding="SAME", kernel_initializer='random_uniform', bias_initializer='zeros', name="layer4")(x) if mode == 'bn': x = KL.BatchNormalization()(x) x = KL.ReLU()(x) x = KL.Conv2D(3, (3, 3), padding="SAME", kernel_initializer='random_uniform', bias_initializer='zeros', name="layer5")(x) if mode == 'bn': x = KL.BatchNormalization()(x) x = KL.ReLU()(x) if mode == 'skip' or mode == 'bn': x = KL.average([x, inputs]) return KM.Model(inputs=inputs, outputs=x, name='denoising')
def create_ensemble(models): if len(models) == 1: return models[0] else: inputs = Input(shape=[N_FEATURES]) predictions = [model(inputs) for model in models] outputs = average(predictions) model = Model(inputs=inputs, outputs=outputs) model.compile( loss='mse', metrics=[RootMeanSquaredError(), MeanAbsoluteError(), RSquare()]) return model
def load_weights(self, atom_type): saved_models_dir = os.path.join(self.path, atom_type) model_0_path = os.path.join(saved_models_dir, 'model_10_n1_8.h5') model_1_path = os.path.join(saved_models_dir, 'model_1_n1_8.h5') model_2_path = os.path.join(saved_models_dir, 'model_2_n1_8.h5') model_3_path = os.path.join(saved_models_dir, 'model_3_n1_8.h5') model_4_path = os.path.join(saved_models_dir, 'model_4_n1_8.h5') model_5_path = os.path.join(saved_models_dir, 'model_5_n1_8.h5') model_6_path = os.path.join(saved_models_dir, 'model_6_n1_8.h5') model_7_path = os.path.join(saved_models_dir, 'model_7_n1_8.h5') model_8_path = os.path.join(saved_models_dir, 'model_8_n1_8.h5') model_9_path = os.path.join(saved_models_dir, 'model_9_n1_8.h5') # Load all the models model_0 = keras.models.load_model(model_0_path, compile=False) model_1 = keras.models.load_model(model_1_path, compile=False) model_2 = keras.models.load_model(model_2_path, compile=False) model_3 = keras.models.load_model(model_3_path, compile=False) model_4 = keras.models.load_model(model_4_path, compile=False) model_5 = keras.models.load_model(model_5_path, compile=False) model_6 = keras.models.load_model(model_6_path, compile=False) model_7 = keras.models.load_model(model_7_path, compile=False) model_8 = keras.models.load_model(model_8_path, compile=False) model_9 = keras.models.load_model(model_9_path, compile=False) inputs = keras.Input(shape=(384, )) # Define each sub model y0 = model_0(inputs) y1 = model_1(inputs) y2 = model_2(inputs) y3 = model_3(inputs) y4 = model_4(inputs) y5 = model_5(inputs) y6 = model_6(inputs) y7 = model_7(inputs) y8 = model_8(inputs) y9 = model_9(inputs) # Finally generate ensemble model ml_corrs = [y0, y1, y2, y3, y4, y5, y6, y7, y8, y9] outputs = layers.average([y0, y1, y2, y3, y4, y5, y6, y7, y8, y9]) print(f"Loaded ensemble {atom_type}") return keras.Model(inputs=inputs, outputs=([outputs, ml_corrs]))
def ensembleModels(models, ): """ This function builds an ensemble model from a list of models Arguments: models: list of Keras models Return: modelEns: ensemble Keras model """ model_input = Input(shape=models[0].input_shape[1:]) # collect outputs of models in a list yModels=[model(model_input) for model in models] # averaging outputs yAvg=layers.average(yModels) # build model from same input and avg output modelEns = Model(inputs=model_input, outputs=yAvg, name='ensemble') return modelEns
def avg_ensemble(model_dir: str): sub_models = [] model_files = Path(model_dir).glob('*.h5') for model in model_files: # skip ensemble model if model.stem == 'ensemble': continue sub_models.append(load_model(model, compile=False)) text_input = layers.Input(shape=(sub_models[0].input['text'].shape[1:]), name='text') kw_input = layers.Input(shape=(sub_models[0].input['keyword'].shape[1:]), name='keyword') loc_input = layers.Input(shape=(sub_models[0].input['location'].shape[1:]), name='location') hashtags_input = layers.Input( shape=(sub_models[0].input['hashtags'].shape[1:]), name='hashtags') inputs = { 'text': text_input, 'keyword': kw_input, 'location': loc_input, 'hashtags': hashtags_input } model_outputs = [] for model in sub_models: # remove last layer which should be the activation layer model.layers.pop(-1) output = model.layers[-1].output model = Model(inputs=model.input, outputs=output) model_outputs.append(model(inputs)) average = layers.average([model for model in model_outputs]) softmax = layers.Activation(tf.nn.softmax)(average) ensemble = Model(inputs=inputs, outputs=softmax) save_model(ensemble, os.path.join(model_dir, 'ensemble.h5')) plot_model(ensemble, to_file=os.path.join(model_dir, 'ensemble.png'), show_shapes=True) return ensemble
for layer in object_mg_model.layers: layer._name = 'object_' + layer.name object_output = object_mg_model.get_layer(name='object_dense_1').output ##------------------------------------------------------------------------------------ ## get layer from places_mg_model (change directory to load best places_mg_model from folder 'training_models') places_mg_model = load_model('./pretrained_models/places_mg_model.hdf5') for layer in places_mg_model.layers: layer._name = 'places_' + layer.name places_output = places_mg_model.get_layer(name='places_dense_2').output ## Average both models output average_layer = average([object_output, places_output]) ## Create late_fusion2_model late_fusion2_model = Model( inputs=[Object_MG_Model.input, Places_MG_Model.input], outputs=average_layer) ground_truth = test_datagen1.classes ## Data generator dgenerator = doubleGenerator(test_datagen1, test_datagen2) ## Predictions predictions = late_fusion2_model.predict_generator( dgenerator, steps=math.ceil(test_datagen1.samples / BATCH_SIZE), verbose=1) predicted_classes = np.argmax(predictions, axis=1)
def get_model(): inputs = keras.Input(shape=(128, )) outputs = keras.layers.Dense(1, activation='sigmoid')(inputs) return keras.Model(inputs, outputs) model1 = get_model() model2 = get_model() model3 = get_model() inputs = keras.Input(shape=(128, )) y1 = model1(inputs) y2 = model2(inputs) y3 = model3(inputs) outputs = layers.average([y1, y2, y3]) ensemble_model = keras.Model(inputs, outputs) #3.复杂网络结构构建 #3.1 多输入与多输出网络 #构建一个根据定制标题、内容和标签, 预测票证优先级和执行部门的网络 #超参数 num_words = 2000 num_tags = 12 num_department = 4 #输入 body_input = keras.Input(shape=(None, ), name='body') title_input = keras.Input(shape=(None, ), name='title') tag_input = keras.Input(shape=(num_tags, ), name='tag')
def _create_multicol_model(): inputs = Input(shape=(None, None, 3)) cols = [_create_column(d, inputs) for d in [3, 5, 9]] model = KModel(inputs=inputs, outputs=average(cols)) return model
def create_object_mg_model(best_object_basic_model): model = load_model(best_object_basic_model) MobileNet_model_out = model.get_layer('conv_pw_1_relu').output conv_1 = Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None)(MobileNet_model_out) branch_1_out = GlobalAveragePooling2D(name='gap1')(conv_1) MobileNet_model_out = model.get_layer('conv_pw_3_relu').output conv_2 = Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None)(MobileNet_model_out) branch_2_out = GlobalAveragePooling2D(name='gap2')(conv_2) MobileNet_model_out = model.get_layer('conv_pw_5_relu').output conv_3 = Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None)(MobileNet_model_out) branch_3_out = GlobalAveragePooling2D(name='gap3')(conv_3) MobileNet_model_out = model.get_layer('conv_pw_11_relu').output conv_4 = Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None)(MobileNet_model_out) branch_4_out = GlobalAveragePooling2D(name='gap4')(conv_4) MobileNet_model_out = model.get_layer('conv_pw_13_relu').output conv_5 = Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None)(MobileNet_model_out) branch_5_out = GlobalAveragePooling2D(name='gap5')(conv_5) merge = average( [branch_1_out, branch_2_out, branch_3_out, branch_4_out, branch_5_out]) output = Dense(8, activation='softmax')(merge) model = Model(inputs=model.input, outputs=[output]) return model
# PermutationalEncoder( # PairwiseModel((8,), repeat_layers(Dense, [16, 4], activation="relu")), 3 # ), # name="permutational_layer2", # ) # outputs = perm_layer2(outputs) # perm_layer3 = PermutationalLayer( # PermutationalEncoder( # PairwiseModel((4,), repeat_layers(Dense, [16, 4], activation="tanh")), 3 # ), # name="permutational_layer3", # ) # outputs = perm_layer3(outputs) # # I can even reuse this layer because the input shape and output shape of it is identical. but don't try this at home unless you know what you are doing. # outputs = perm_layer3(outputs) output = average(outputs) # let's average the output for single tensor model = Model(inputs, output) print("# Multi-layer model summary") model.summary() # let's predict with the big mult layers model on some similar data print("# Multi-layer Prediction") print( "Because the layer is preserving permutation invariance. " "The output will be the same no matter how many times you permute the input order." ) print("## Input/Output 1") x = [[[0, 0, 0, 0]], [[1, 2, 3, 4]], [[4, 3, 2, 1]]] print(x) print(model.predict(x)) print("## Input/Output 2")
def CNorm(vst_onlyTokens, dl_terms, dl_associations, vso, nbEpochs=30, batchSize=64, l_numberOfFilters=[4000], l_filterSizes=[1], phraseMaxSize=15): # Preparing data for SLFNN and S-CNN components: dataSCNN, labels, l_unkownTokens, l_uncompleteExpressions = prepare2D_data( vst_onlyTokens, dl_terms, dl_associations, vso, phraseMaxSize) dataSLFNN = numpy.zeros((dataSCNN.shape[0], dataSCNN.shape[2])) for i in range(dataSCNN.shape[0]): numberOfToken = 0 for embedding in dataSCNN[i]: if not numpy.any(embedding): pass else: numberOfToken += 1 dataSLFNN[i] += embedding if numberOfToken > 0: dataSLFNN[i] = dataSLFNN[i] / numberOfToken # Input layers: inputLP = Input(shape=dataSLFNN.shape[1]) inputCNN = Input(shape=[dataSCNN.shape[1], dataSCNN.shape[2]]) # SLFNN component: ontoSpaceSize = labels.shape[2] denseLP = layers.Dense( units=ontoSpaceSize, use_bias=True, kernel_initializer=initializers.GlorotUniform())(inputLP) modelLP = Model(inputs=inputLP, outputs=denseLP) # Shallow-CNN component: l_subLayers = list() for i, filterSize in enumerate(l_filterSizes): convLayer = (layers.Conv1D( l_numberOfFilters[i], filterSize, strides=1, kernel_initializer=initializers.GlorotUniform()))(inputCNN) outputSize = phraseMaxSize - filterSize + 1 pool = (layers.MaxPool1D(pool_size=outputSize))(convLayer) activationLayer = (layers.LeakyReLU(alpha=0.3))(pool) l_subLayers.append(activationLayer) if len(l_filterSizes) > 1: concatenateLayer = (layers.Concatenate(axis=-1))( l_subLayers) # axis=-1 // concatenating on the last dimension else: concatenateLayer = l_subLayers[0] denseLayer = layers.Dense( ontoSpaceSize, kernel_initializer=initializers.GlorotUniform())(concatenateLayer) modelCNN = Model(inputs=inputCNN, outputs=denseLayer) convModel = Model(inputs=inputCNN, outputs=concatenateLayer) fullmodel = models.Sequential() fullmodel.add(convModel) # Combination of the two components: combinedLayer = layers.average([modelLP.output, modelCNN.output]) fullModel = Model(inputs=[inputLP, inputCNN], outputs=combinedLayer) fullModel.summary() # Compile and train: fullModel.compile( optimizer=optimizers.Nadam(), loss=losses.LogCosh(), metrics=[metrics.CosineSimilarity(), metrics.MeanSquaredError()]) fullModel.fit([dataSLFNN, dataSCNN], labels, epochs=nbEpochs, batch_size=batchSize) return fullModel, vso, l_unkownTokens
def _create_msb(filters, dimensions, inputs): """ Multi-scale Blob as described in https://arxiv.org/pdf/1702.02359.pdf """ cols = [Conv2D(filters, kernel_size=(d, d), activation='relu', padding='same', kernel_initializer=_msb_initializer)(inputs) for d in dimensions] return average(cols)
def MT_model(perm): #def funx1(i,maxL): #if i<maxL: #return Input(shape=(fpSize,),name=f"MRg_{i}") #elif i<2*maxL-3 and i!=maxL+2: # elif maxL<=i< 2*maxL: #return Input(shape=(FpLen,),name=f"Ml_{i}") #else: # return Input(shape=(FpLen1,),name=f"Sq_{i}") def funx1(i, maxL): if i < maxL: return Input(shape=(256, ), name=f"MoR_{i}") x2 = [funx1(i, maxL) for i in range(1 * maxL)] # x2_1=Concatenate()([x2[0],x2[1],x2[2]]) xs = [] xs = [x2[perm[0]], x2[perm[1]], x2[perm[2]]] #xs=[x2[1],x2[2],x2[0]] #xs=[x2[1],x2[0],x2[2]] #xs=[x2[2],x2[0],x2[1]] #xs=[x2[2],x2[1],x2[0]] #xs=[x2[2],x2[0],x2[1]] #xs1=[] xs2 = [] #xs.append(x2) #for i in range(maxL): # xs.append(x2[i]) # xs.append(x2[maxL+i]) # xs.append(x2[2*maxL+i]) list1 = [100, 100, 100, 100] #list1=[100,58,58,58] # list2=[112,112,112,112] pairwise_model = pl.PairwiseModel((256, ), pl.repeat_layers(Dense, list1, name="hidden", activation='relu'), name="pairwise_model") perm_encoder = pl.PermutationalEncoder(pairwise_model, maxL, name="permutational_encoder") perm_layer = pl.PermutationalLayer1(perm_encoder, name="permutational_layer") outputs = perm_layer.model(xs) outputs = average(outputs) #outputs = maximum(outputs) # print("x2",x2) output_51 = Dense(100, activation='relu', kernel_initializer=my_init)(outputs) # output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51) # output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51) # output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51) # output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51) # output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51) # #output_51 = Dropout(0.5)(output_51) # output_51 = Dense(200, activation='relu',kernel_initializer=my_init)(output_51) # output_Loss = Dense(17, name='Loss_output', activation='softmax', kernel_initializer=my_init)(output_51) # output_Loss=Dense(17,name='Loss_output',activation='linear',kernel_initializer=my_init)(output_51) #output_Loss=Dense(17,name='Loss_output',activation='linear',kernel_initializer=my_init)(output_51) model = Model(inputs=x2, outputs=output_Loss) # model.compile(loss={'Loss_output':'categorical_crossentropy'}, # optimizer=Adam(lr=0.00005, beta_1=0.9, beta_2=0.999, epsilon=1e-8), loss_weights = {'Loss_output':1.} # ,metrics=['accuracy'] # ) model.compile( loss={'Loss_output': 'categorical_crossentropy'}, optimizer=Adam(lr=0.00276, beta_1=0.9, beta_2=0.999, epsilon=1e-8), loss_weights={'Loss_output': 1.} #optimizer=Adam(lr=0.00005, beta_1=0.9, beta_2=0.999, epsilon=1e-8), loss_weights = {'Loss_output':1.} , metrics=['accuracy']) model.summary() plot_model(model, 'Config3Mod.png', show_shapes=True) return model
def MNet(input_shape): """Code copied from `Model_MNet <https://github.com/HzFu/MNet_DeepCDR/blob/master/mnet_deep_cdr/Model_MNet.py>`_. Made some minor changes to adapt the network to our specific case. Based on `Joint Optic Disc and Cup Segmentation Based on Multi-label Deep Network and Polar Transformation <https://arxiv.org/abs/1801.00926>`_. Parameters ---------- input_shape : 3 int tuple Shape of input images. Returns ------- model : Keras model MNet network model. Here is a picture of the network extracted from the original paper: .. image:: img/MNet.jpg :width: 100% :align: center """ img_input = Input(input_shape) scale_img_2 = AveragePooling2D(pool_size=(2, 2))(img_input) scale_img_3 = AveragePooling2D(pool_size=(2, 2))(scale_img_2) scale_img_4 = AveragePooling2D(pool_size=(2, 2))(scale_img_3) conv1 = Conv2D(32, (3, 3), padding='same', activation='relu', name='block1_conv1')(img_input) conv1 = Conv2D(32, (3, 3), padding='same', activation='relu', name='block1_conv2')(conv1) pool1 = MaxPooling2D(pool_size=(2, 2))(conv1) input2 = Conv2D(64, (3, 3), padding='same', activation='relu', name='block2_input1')(scale_img_2) input2 = concatenate([input2, pool1], axis=3) conv2 = Conv2D(64, (3, 3), padding='same', activation='relu', name='block2_conv1')(input2) conv2 = Conv2D(64, (3, 3), padding='same', activation='relu', name='block2_conv2')(conv2) pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) input3 = Conv2D(128, (3, 3), padding='same', activation='relu', name='block3_input1')(scale_img_3) input3 = concatenate([input3, pool2], axis=3) conv3 = Conv2D(128, (3, 3), padding='same', activation='relu', name='block3_conv1')(input3) conv3 = Conv2D(128, (3, 3), padding='same', activation='relu', name='block3_conv2')(conv3) pool3 = MaxPooling2D(pool_size=(2, 2))(conv3) input4 = Conv2D(256, (3, 3), padding='same', activation='relu', name='block4_input1')(scale_img_4) input4 = concatenate([input4, pool3], axis=3) conv4 = Conv2D(256, (3, 3), padding='same', activation='relu', name='block4_conv1')(input4) conv4 = Conv2D(256, (3, 3), padding='same', activation='relu', name='block4_conv2')(conv4) pool4 = MaxPooling2D(pool_size=(2, 2))(conv4) conv5 = Conv2D(512, (3, 3), padding='same', activation='relu', name='block5_conv1')(pool4) conv5 = Conv2D(512, (3, 3), padding='same', activation='relu', name='block5_conv2')(conv5) up6 = concatenate([ Conv2DTranspose( 256, (2, 2), strides=(2, 2), padding='same', name='block6_dconv')(conv5), conv4 ], axis=3) conv6 = Conv2D(256, (3, 3), padding='same', activation='relu', name='block6_conv1')(up6) conv6 = Conv2D(256, (3, 3), padding='same', activation='relu', name='block6_conv2')(conv6) up7 = concatenate([ Conv2DTranspose( 128, (2, 2), strides=(2, 2), padding='same', name='block7_dconv')(conv6), conv3 ], axis=3) conv7 = Conv2D(128, (3, 3), padding='same', activation='relu', name='block7_conv1')(up7) conv7 = Conv2D(128, (3, 3), padding='same', activation='relu', name='block7_conv2')(conv7) up8 = concatenate([ Conv2DTranspose( 64, (2, 2), strides=(2, 2), padding='same', name='block8_dconv')(conv7), conv2 ], axis=3) conv8 = Conv2D(64, (3, 3), padding='same', activation='relu', name='block8_conv1')(up8) conv8 = Conv2D(64, (3, 3), padding='same', activation='relu', name='block8_conv2')(conv8) up9 = concatenate([ Conv2DTranspose( 32, (2, 2), strides=(2, 2), padding='same', name='block9_dconv')(conv8), conv1 ], axis=3) conv9 = Conv2D(32, (3, 3), padding='same', activation='relu', name='block9_conv1')(up9) conv9 = Conv2D(32, (3, 3), padding='same', activation='relu', name='block9_conv2')(conv9) side6 = UpSampling2D(size=(8, 8))(conv6) side7 = UpSampling2D(size=(4, 4))(conv7) side8 = UpSampling2D(size=(2, 2))(conv8) # Changes here to have just one class and use BCE as loss function out6 = Conv2D(1, (1, 1), activation='sigmoid', name='side_63')(side6) out7 = Conv2D(1, (1, 1), activation='sigmoid', name='side_73')(side7) out8 = Conv2D(1, (1, 1), activation='sigmoid', name='side_83')(side8) out9 = Conv2D(1, (1, 1), activation='sigmoid', name='side_93')(conv9) out10 = average([out6, out7, out8, out9]) return Model(inputs=[img_input], outputs=[out6, out7, out8, out9, out10])
def call(self, x): # three scale imgs scale_img_1 = self.scale1(x) scale_img_2 = self.scale2(scale_img_1) scale_img_3 = self.scale3(scale_img_2) conv1 = self.block1_conv1(x) conv1 = self.block1_conv2(conv1) pool1 = self.block1_pool1(conv1) input2 = self.block2_input1(scale_img_1) input2 = tf.concat([input2, pool1], axis=3) conv2 = self.block2_conv1(input2) conv2 = self.block2_conv2(conv2) pool2 = self.block2_pool1(conv2) input3 = self.block3_input1(scale_img_2) input3 = tf.concat([input3, pool2], axis=3) conv3 = self.block3_conv1(input3) conv3 = self.block3_conv2(conv3) pool3 = self.block3_pool1(conv3) input4 = self.block4_input1(scale_img_3) input4 = tf.concat([input4, pool3], axis=3) conv4 = self.block4_conv1(input4) conv4 = self.block4_conv2(conv4) pool4 = self.block4_pool1(conv4) conv5 = self.block5_conv1(pool4) conv5 = self.block5_conv2(conv5) temp6 = self.block6_dconv(conv5) up6 = tf.concat([temp6, conv4], axis=3) conv6 = self.block6_conv1(up6) conv6 = self.block6_conv2(conv6) temp7 = self.block7_dconv(conv6) up7 = tf.concat([temp7, conv3], axis=3) conv7 = self.block7_conv1(up7) conv7 = self.block7_conv2(conv7) temp8 = self.block8_dconv(conv7) up8 = tf.concat([temp8, conv2], axis=3) conv8 = self.block8_conv1(up8) conv8 = self.block8_conv2(conv8) temp9 = self.block9_dconv(conv8) up9 = tf.concat([temp9, conv1], axis=3) conv9 = self.block9_conv1(up9) conv9 = self.block9_conv2(conv9) side6 = UpSampling2D(size=(8, 8))(conv6) side7 = UpSampling2D(size=(4, 4))(conv7) side8 = UpSampling2D(size=(2, 2))(conv8) out6 = self.side63(side6) out7 = self.side73(side7) out8 = self.side83(side8) out9 = self.side93(conv9) out10 = average([out6, out7, out8, out9]) return [out6, out7, out8, out9, out10]
def prepare_encoder_output_state(self, inputs): if 'encoder_output_state' in inputs: encoder_output_state = inputs['encoder_output_state'] else: hidden = inputs['hidden'] if len(hidden.shape) == 3: # encoder_output is a sequence # reduce_sequence returns a [b, h] encoder_output_state = self.reduce_sequence(hidden) elif len(hidden.shape) == 2: # this returns a [b, h] encoder_output_state = hidden else: raise ValueError("Only works for 1d or 2d encoder_output") # now we have to deal with the fact that the state needs to be a list # in case of lstm or a tensor otherwise if (self.cell_type == 'lstm' and isinstance(encoder_output_state, list)): if len(encoder_output_state) == 2: # this maybe a unidirectionsl lstm or a bidirectional gru / rnn # there is no way to tell # If it is a unidirectional lstm, pass will work fine # if it is bidirectional gru / rnn, the output of one of # the directions will be treated as the inital c of the lstm # which is weird and may lead to poor performance # todo future: try to find a way to distinguish among these two cases pass elif len(encoder_output_state) == 4: # the encoder was a bidirectional lstm # a good strategy is to average the 2 h and the 2 c vectors encoder_output_state = [ average( [encoder_output_state[0], encoder_output_state[2]] ), average( [encoder_output_state[1], encoder_output_state[3]] ) ] else: # no idea how lists of length different than 2 or 4 # might have been originated, we can either rise an ValueError # or deal with it averaging everything # raise ValueError( # "encoder_output_state has length different than 2 or 4. " # "Please doublecheck your encoder" # ) average_state = average(encoder_output_state) encoder_output_state = [average_state, average_state] elif (self.cell_type == 'lstm' and not isinstance(encoder_output_state, list)): encoder_output_state = [encoder_output_state, encoder_output_state] elif (self.cell_type != 'lstm' and isinstance(encoder_output_state, list)): # here we have a couple options, # either reuse part of the input encoder state, # or just use its output if len(encoder_output_state) == 2: # using h and ignoring c encoder_output_state = encoder_output_state[0] elif len(encoder_output_state) == 4: # using average of hs and ignoring cs encoder_output_state + average( [encoder_output_state[0], encoder_output_state[2]] ) else: # no idea how lists of length different than 2 or 4 # might have been originated, we can either rise an ValueError # or deal with it averaging everything # raise ValueError( # "encoder_output_state has length different than 2 or 4. " # "Please doublecheck your encoder" # ) encoder_output_state = average(encoder_output_state) # this returns a [b, h] # decoder_input_state = reduce_sequence(eo, self.reduce_input) elif (self.cell_type != 'lstm' and not isinstance(encoder_output_state, list)): # do nothing, we are good pass # at this point decoder_input_state is either a [b,h] # or a list([b,h], [b,h]) if the decoder cell is an lstm # but h may not be the same as the decoder state size, # so we may need to project if isinstance(encoder_output_state, list): for i in range(len(encoder_output_state)): if (encoder_output_state[i].shape[1] != self.state_size): encoder_output_state[i] = self.project( encoder_output_state[i] ) else: if encoder_output_state.shape[1] != self.state_size: encoder_output_state = self.project( encoder_output_state ) return encoder_output_state
def main(): arg_list = None args = parseArgs(arg_list) # grab training data filepath = 'data/train_sample_videos' datapath = os.path.join(filepath, 'metadata.json') data = pd.read_json(datapath).T if args.sample: files = [os.path.join(filepath, f) for f in data.index][:20] labels = data.label.values[:20] else: files = [os.path.join(filepath, f) for f in data.index] labels = data.label.values x_train, x_test, y_train, y_test = train_test_split( files, labels, test_size=float(args.test_split)) class_weights = compute_class_weight( 'balanced', np.unique(y_train), y_train) for k, v in zip(np.unique(y_train), class_weights): print(k, v) y_train = list(map(lambda x: 0 if x == 'REAL' else 1, y_train)) y_test = list(map(lambda x: 0 if x == 'REAL' else 1, y_test)) y_train = to_categorical(y_train, num_classes=2) y_test = to_categorical(y_test, num_classes=2) print(len(x_train), len(y_train), len(x_test), len(y_test)) # validation data val_path = 'data/test_videos' if args.sample: val_files = [os.path.join(val_path, f) for f in os.listdir(val_path)][:8] else: val_files = [os.path.join(val_path, f) for f in os.listdir(val_path)] print('number of validation files', len(val_files)) # generate datasets batch_size = args.batch_size segment_size = args.segment_size rsz = (128, 128) train_data = input_fn( x_train, y_train, segment_size=segment_size, batch_size=batch_size, rsz=rsz) test_data = input_fn( x_test, y_test, segment_size=segment_size, batch_size=batch_size, rsz=rsz) val_data = input_fn( files=val_files, segment_size=segment_size, batch_size=batch_size, rsz=rsz) rgb_input = tf.keras.Input( shape=(segment_size, rsz[0], rsz[1], 3), name='rgb_input') flow_input = tf.keras.Input( shape=(segment_size - 1, rsz[0], rsz[1], 2), name='flow_input') # TODO: make OO # RGB MODEL # block 1 x = layers.Conv3D( filters=8, kernel_size=3, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(rgb_input) x = layers.Conv3D( filters=8, kernel_size=4, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(x) block1_output = layers.MaxPool3D( pool_size=(2, 2, 2), strides=(2, 2, 2), padding='same' )(x) # block 2 x = layers.Conv3D( filters=8, kernel_size=3, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(block1_output) x = layers.Conv3D( filters=8, kernel_size=4, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(x) block2_output = layers.add([x, block1_output]) # block 3 x = layers.Conv3D( filters=8, kernel_size=3, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(block2_output) x = layers.Conv3D( filters=8, kernel_size=4, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(x) block3_output = layers.add([x, block2_output]) x = layers.Conv3D( filters=8, kernel_size=3, strides=(1, 1, 1), padding='same', data_format='channels_last', activation='relu', )(block3_output) x = layers.GlobalAveragePooling3D()(x) x = layers.Dense(64, activation='relu')(x) x = layers.Dropout(0.5)(x) rgb_outputs = layers.Dense(2, activation='softmax')(x) rgb_model = Model(inputs=rgb_input, outputs=rgb_outputs) rgb_model.summary() # FLOW MODEL x = layers.ConvLSTM2D( filters=8, kernel_size=3, strides=1, padding='same', data_format='channels_last', return_sequences=True, dropout=0.5 )(flow_input) x = layers.BatchNormalization()(x) x = layers.ConvLSTM2D( filters=8, kernel_size=3, strides=1, padding='same', data_format='channels_last', return_sequences=True, dropout=0.5 )(x) x = layers.BatchNormalization()(x) x = layers.ConvLSTM2D( filters=8, kernel_size=3, strides=1, padding='same', data_format='channels_last', return_sequences=False, dropout=0.5 )(x) x = layers.BatchNormalization()(x) x = layers.Flatten()(x) x = layers.Dense(128, activation='relu')(x) x = layers.Dense(128, activation='relu')(x) x = layers.Dense(128, activation='relu')(x) x = layers.Dropout(0.5)(x) flow_output = layers.Dense(2)(x) flow_model = Model(inputs=flow_input, outputs=flow_output) flow_model.summary() # FINAL MODEL final_average = layers.average([rgb_outputs, flow_output]) x = layers.Flatten()(final_average) final_output = layers.Dense( 2, activation='softmax', name='final_output')(x) model = Model( inputs={"rgb_input": rgb_input, "flow_input": flow_input}, outputs=final_output, name='my_model' ) model.summary() # tf.keras.utils.plot_model( # model, # to_file='model.png', # show_shapes=True, # show_layer_names=True # ) # TRAIN dt = datetime.now().strftime('%Y%m%d_%H%M%S') opt = tf.keras.optimizers.Adam() if args.save_checkpoints: save_path = f'data/model_checkpoints/{dt}/ckpt' ckpt = tf.keras.callbacks.ModelCheckpoint( filepath=save_path, save_best_only=False, save_weights_only=True ) ckpt = [ckpt] else: ckpt = [] model.compile( optimizer=opt, loss='categorical_crossentropy', metrics=['acc']) model.fit( x=train_data.repeat(), validation_data=test_data.repeat(), epochs=args.epochs, verbose=1, class_weight=class_weights, steps_per_epoch=len(x_train) // batch_size, validation_steps=len(x_test) // batch_size, callbacks=ckpt ) # EVAL print('\n\n---------------------------------------------------------') print('predicting on validation data') start = time.time() preds = model.predict( val_data, verbose=1, steps=len(val_files) // batch_size ) print('prediction time: ', time.time() - start) preds = np.argmax(preds, axis=1) df = pd.DataFrame(columns=['filename', 'label']) df.filename = [v.split('/')[-1] for v in val_files] df.label = preds df.to_csv(f'data/submission_{dt}.csv', index=False)