def NASNet_ensemble_FCN(input_image, weights=None, fine_tune=False): input_tensor = Input(shape=(input_image)) model = NASNetMobile(input_shape=(224, 224, 3), input_tensor=input_tensor, include_top=False, weights=weights) #print model.summary() #return #For fine-tuning if fine_tune: nasnet_layers = model.layers for layer in nasnet_layers: print layer layer.trainable = False stem_2 = model.get_layer(name='reduction_concat_stem_2').output reduce_4 = model.get_layer(name='reduction_concat_reduce_4').output normal_12 = model.get_layer(name='normal_concat_12').output #fcn_8s conv_normal_12 = Conv2D(filters=6, kernel_size=(1, 1))(normal_12) upscore_normal_12 = Conv2DTranspose(filters=6, kernel_size=(4, 4), strides=(2, 2), padding='same')(conv_normal_12) conv_reduce_4 = Conv2D(filters=6, kernel_size=(1, 1))(reduce_4) fuse_4 = Add()([conv_reduce_4, upscore_normal_12]) upscore_fuse_4 = Conv2DTranspose(filters=6, kernel_size=(4, 4), strides=(2, 2), padding='same')(fuse_4) conv_stem_2 = Conv2D(filters=6, kernel_size=(1, 1))(stem_2) fuse_2 = Add()([conv_stem_2, upscore_fuse_4]) output_8 = Conv2DTranspose(filters=6, kernel_size=(16, 16), strides=(8, 8), padding='same')(fuse_2) output_16 = Conv2DTranspose(filters=6, kernel_size=(32, 32), strides=(16, 16), padding='same')(fuse_4) #fcn_32s output_32 = Conv2DTranspose(filters=6, kernel_size=(64, 64), strides=(32, 32), padding='same')(conv_normal_12) model = Model(inputs=input_tensor, outputs=[output_8, output_16, output_32]) print model.summary() return model
def save_model11(new_model_path, conv_model_path): model = NASNetMobile( input_shape=(img_width, img_height, 3), include_top=False, weights=None ) if pretrained: model = NASNetMobile( input_shape=(img_width, img_height, 3), include_top=False, weights='imagenet' ) model.summary() transfer_layer = model.get_layer('?') conv_model = Model(inputs=model.input, outputs=transfer_layer.output) new_model = Sequential() new_model.add(conv_model) new_model.add(GlobalAveragePooling2D()) if num_fc_layers>=1: new_model.add(Dense(num_fc_neurons, activation='relu')) if num_fc_layers>=2: new_model.add(Dropout(dropout)) new_model.add(Dense(num_fc_neurons, activation='relu')) if num_fc_layers>=3: new_model.add(Dropout(dropout)) new_model.add(Dense(num_fc_neurons, activation='relu')) new_model.add(Dense(num_classes, activation='softmax')) print(new_model.summary()) new_model.save(new_model_path) conv_model.save(conv_model_path) return
def __init__(self, cnnModel, weights=None): K.clear_session() self.cnnModel = cnnModel self.weights = weights # so we can check elsewhere which model if weights is None: if cnnModel == 'InceptionV3': # Get model with pretrained weights. base_model = InceptionV3(weights='imagenet', include_top=True) # We'll extract features at the final pool layer. self.model = Model( inputs=base_model.input, outputs=base_model.get_layer('avg_pool').output) self.target_size = (299, 299) self.preprocess_input = inception_v3_preprocessor elif cnnModel == 'VGG19': # Get model with pretrained weights. base_model = VGG19(weights='imagenet', include_top=True) # We'll extract features at the final pool layer. self.model = Model(inputs=base_model.input, outputs=base_model.get_layer('fc2').output) self.target_size = (224, 224) self.preprocess_input = vgg19_preprocessor elif cnnModel == 'InceptionResNetV2': # Get model with pretrained weights. base_model = InceptionResNetV2(weights='imagenet', include_top=True) # We'll extract features at the final pool layer. self.model = Model( inputs=base_model.input, outputs=base_model.get_layer('avg_pool').output) self.target_size = (299, 299) self.preprocess_input = inception_resnet_v2_preprocessor elif cnnModel == 'NASNetMobile': # Get model with pretrained weights. base_model = NASNetMobile(weights='imagenet', include_top=True) # We'll extract features at the final pool layer. self.model = Model(inputs=base_model.input, outputs=base_model.get_layer( 'global_average_pooling2d_1').output) self.target_size = (224, 224) self.preprocess_input = nasnet_preprocessor elif cnnModel == 'NASNetMobileCropTo11': # Get model with pretrained weights. base_model = NASNetMobile(weights='imagenet', include_top=True) # We'll extract features at the final pool layer. interModel = Model( inputs=base_model.input, outputs=base_model.get_layer('activation_188').output) self.model = Sequential() self.model.add(interModel) self.model.add(Cropping2D(cropping=((3, 3), (3, 3)))) self.model.add(Flatten()) self.target_size = (224, 224) self.preprocess_input = nasnet_preprocessor elif cnnModel == 'NASNetMobileCropTo33': base_model = NASNetMobile(weights='imagenet', include_top=True) # We'll extract features at the final pool layer. interModel = Model( inputs=base_model.get_layer(index=0).input, outputs=base_model.get_layer(index=-3).output) self.model = Sequential() self.model.add(interModel) self.model.add(Cropping2D(cropping=((2, 2), (2, 2)))) self.model.add(Flatten()) self.target_size = (224, 224) self.preprocess_input = nasnet_preprocessor elif cnnModel == 'NASNetMobileOLD': # Get model with pretrained weights. base_model = NASNetMobile(weights='imagenet', include_top=False, pooling='none') self.model = base_model self.target_size = (224, 224) self.preprocess_input = nasnet_preprocessor elif cnnModel == 'NASNetLarge': # Get model with pretrained weights. base_model = NASNetLarge(weights='imagenet', include_top=True) # We'll extract features at the final pool layer. self.model = Model(inputs=base_model.input, outputs=base_model.get_layer( 'global_average_pooling2d_1').output) self.target_size = (331, 331) self.preprocess_input = nasnet_preprocessor elif cnnModel == 'cifar10vgg': base_model = cifar10vgg(False) interModel = Model( inputs=base_model.model.input, # outputs=base_model.model.get_layer('flatten_1').output outputs=base_model.model.get_layer( 'max_pooling2d_3').output # outputs=base_model.model.get_layer('dropout_10').output # outputs=base_model.model.get_layer('max_pooling2d_3').output ) self.model = Sequential() self.model.add(interModel) self.model.add(Flatten()) self.target_size = (32, 32) else: # Load the model first. self.model = load_model(weights) # Then remove the top so we get features not predictions. # From: https://github.com/fchollet/keras/issues/2371 self.model.layers.pop() self.model.layers.pop() # two pops to get to pool layer self.model.outputs = [self.model.layers[-1].output] self.model.output_layers = [self.model.layers[-1]] self.model.layers[-1].outbound_nodes = []
class FeatureExtraction(): """ Feature Extraction class for extracting features from image data through pre-trained model and saving features and labels to user defined path. """ def __init__(self, model="mobilenet", weights="imagenet", include_top=True): if model_name == "vgg16": self.base_model = VGG16(weights=weights) self.model = Model(input=self.base_model.input, output=self.base_model.get_layer('fc1').output) self.image_size = (224, 224) elif model_name == "vgg19": self.base_model = VGG19(weights=weights) self.model = Model(input=self.base_model.input, output=self.base_model.get_layer('fc1').output) self.image_size = (224, 224) elif model_name == "resnet50": self.base_model = ResNet50(weights=weights) self.base_model.summary() self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('fc1000').output) self.image_size = (224, 224) elif model_name == "inceptionv3": self.base_model = InceptionV3(include_top=include_top, weights=weights, input_tensor=Input(shape=(299,299,3))) self.model = Model(input=self.base_model.input, output=self.base_model.get_layer('custom').output) self.image_size = (299, 299) elif model_name == "inceptionresnetv2": self.base_model = InceptionResNetV2(include_top=include_top, weights=weights, input_tensor=Input(shape=(299,299,3))) self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('custom').output) self.image_size = (299, 299) elif model_name == "mobilenet": self.base_model = MobileNet(include_top=include_top, weights=weights, input_tensor=Input(shape=(224,224,3)), input_shape=(224,224,3)) self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('conv_pw_13_relu').output) self.image_size = (224, 224) elif model_name == "mobilenetv2": self.base_model = MobileNeV2(include_top=include_top, weights=weights, input_tensor=Input(shape=(224,224,3)), input_shape=(224,224,3)) self.base_model.summary() self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('conv_pw_13_relu').output) self.image_size = (224, 224) elif model_name == "xception": self.base_model = Xception(weights=weights, , input_tensor=Input(shape=(299,299,3))) self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('avg_pool').output) self.image_size = (299, 299) elif model_name == "densenet": self.base_model = DenseNet121(include_top=include_top, weights=weights, input_tensor=Input(shape=(224,224,3)), input_shape=(224,224,3)) self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('avg_pool').output) self.image_size = (224, 224) elif model_name == "xception": self.base_model = NASNetMobile(include_top=include_top, weights=weights, input_tensor=Input(shape=(224,224,3)), input_shape=(224,224,3)) self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('avg_pool').output) self.image_size = (224, 224) else: self.base_model = None self.features = [] self.labels = [] def list_directory(self, path=None): train_labels = os.listdir(path) return def label_encoder(self, train_labels=None): le = LabelEncoder() le.fit(train_labels) self.le_labels = le.transform(self.labels) return le, self.le_labels def extract_feature(self, features_path='', train_path=path, train_labels=train_labels): # loop over all the labels in the folder count = 0 for i, label in enumerate(train_labels): cur_path = train_path + "/" + label for image_path in glob.glob(cur_path + "/*.png"): flat = process_image(image_path) features.append(flat) labels.append(label) count += 1 print ("[INFO] processed - " + str(count)) print ("[STATUS] training labels: {}".format(le_labels)) print ("[STATUS] training labels shape: {}".format(le_labels.shape)) def process_image(self, image_path=path): # extract features from images img = image.load_img(image_path, target_size=self.image_size) img = image.img_to_array(img) img = np.expand_dims(img, axis=0) img = preprocess_input(img) feature = model.predict(img) flat = feature.flatten() return flat
def NASNetMobile_enc_dec(H=256, W=256, weights='imagenet', ninchannel=3, noutchannel=1, isregression=True, ismuticlass=False, sdo=False, sdo_frac=0.2): """ This is an encoder-decoder network architecture for image reconstruction and image segmentation based on pretrained NASNetLarge model as encoder. The deocder need to be trained :param img_sz: size of the input image [n x n x 3] :param noutchannel: The number of output channels of the network, only valid if ismuticlass==True else ignored :param isregression: If ==True then model output is single channel without any activation applied to the last layer, ismuticlass and noutchannel is ignored If ==False and ismuticlass==False, then output is single channel and sigmoid activation applied to the last layer If ==False and ismuticlass==True, then output is multichannel determined by noutchannel and softmax activation is appled to the last layer :param ismuticlass: If ==False then output is single channel and sigmoid activation applied to the last layer If ==True then output is multichannel determined by noutchannel and softmax activation is appled to the last layer :return: Encoder Decoder Model """ model = NASNetMobile(include_top=False, weights=weights, input_tensor=None, input_shape=(H, W, 3), pooling=None) # ----- Decoder Network ------ # nFeat_d = 256 convT_0_0_d = UpSampling2D_BN_Act( kSize=2, crop=0, outPad=0, concateInp=add([ model.get_layer('activation_129').output, model.get_layer('activation_130').output ]), inp=model.get_layer('activation_188').output) conv_0_0_d = conv2D_BN_Act(nFeature=nFeat_d, kSize=3, kStride=1, inp=convT_0_0_d, sdo=sdo, sdo_frac=sdo_frac) conv_0_1_d = conv2D_BN_Act(nFeature=nFeat_d, kSize=3, kStride=1, inp=conv_0_0_d, sdo=sdo, sdo_frac=sdo_frac) convT_1_0_d = UpSampling2D_BN_Act( kSize=2, crop=0, outPad=0, concateInp=add([ model.get_layer('activation_70').output, model.get_layer('activation_71').output ]), inp=conv_0_1_d) conv_1_0_d = conv2D_BN_Act(nFeature=nFeat_d, kSize=3, kStride=1, inp=convT_1_0_d, sdo=sdo, sdo_frac=sdo_frac) conv_1_1_d = conv2D_BN_Act(nFeature=nFeat_d, kSize=3, kStride=1, inp=conv_1_0_d, sdo=sdo, sdo_frac=sdo_frac) convT_2_0_d = UpSampling2D_BN_Act( kSize=2, crop=0, outPad=0, concateInp=add([ model.get_layer('activation_13').output, model.get_layer('activation_15').output, model.get_layer('activation_17').output, model.get_layer('activation_19').output ]), inp=conv_1_1_d) conv_2_0_d = conv2D_BN_Act(nFeature=nFeat_d, kSize=3, kStride=1, inp=convT_2_0_d, sdo=sdo, sdo_frac=sdo_frac) conv_2_1_d = conv2D_BN_Act(nFeature=nFeat_d, kSize=3, kStride=1, inp=conv_2_0_d, sdo=sdo, sdo_frac=sdo_frac) convT_3_0_d = UpSampling2D_BN_Act( kSize=2, crop=((0, 1), (0, 1)), outPad=0, concateInp=add([ model.get_layer('activation_1').output, model.get_layer('activation_4').output, model.get_layer('activation_6').output, model.get_layer('activation_8').output ]), inp=conv_2_1_d) conv_3_0_d = conv2D_BN_Act(nFeature=nFeat_d, kSize=3, kStride=1, inp=convT_3_0_d, sdo=sdo, sdo_frac=sdo_frac) conv_3_1_d = conv2D_BN_Act(nFeature=nFeat_d, kSize=3, kStride=1, inp=conv_3_0_d, sdo=sdo, sdo_frac=sdo_frac) convT_4_0_d = UpSampling2D_BN_Act(kSize=2, crop=0, outPad=1, concateInp=model.input, inp=conv_3_1_d) conv_4_0_d = conv2D_BN_Act(nFeature=nFeat_d / 2, kSize=3, kStride=1, inp=convT_4_0_d, sdo=sdo, sdo_frac=sdo_frac) if (isregression): conv_4_1_d = Conv2D(filters=1, kernel_size=3, strides=1, padding='same')(conv_4_0_d) else: if (ismuticlass): conv_4_1_d = Conv2D(filters=noutchannel, kernel_size=3, strides=1, padding='same')(conv_4_0_d) conv_4_1_d = Activation('softmax')(conv_4_1_d) else: conv_4_1_d = Conv2D(filters=1, kernel_size=3, strides=1, padding='same')(conv_4_0_d) conv_4_1_d = Activation('sigmoid')(conv_4_1_d) model_1 = Model(inputs=[model.input], outputs=[conv_4_1_d]) if ninchannel != 3: model_1 = append_inputchannels(model_1, H, W, ninchannel) return model_1
def Build_model(self): ############# Feature Extraction Network ################## if self.Backbone_model == "VGG16": Backbone = VGG16(input_shape=(352,352,3), include_top=False, weights='imagenet') Stages=[Backbone.get_layer("block5_conv3").output, Backbone.get_layer("block4_conv3").output, Backbone.get_layer("block3_conv3").output] # # # # # # # # # # # # # # # # # # # # # # # # # # # # elif self.Backbone_model == "ResNet50": Backbone = ResNet50(input_shape=(352,352,3), include_top=False, weights='imagenet') Stages=[Backbone.output, Backbone.get_layer("conv4_block6_out").output, Backbone.get_layer("conv3_block4_out").output, Backbone.get_layer("conv2_block3_out").output] # # # # # # # # # # # # # # # # # # # # # # # # # # # # elif self.Backbone_model == "NASNetMobile": Backbone = NASNetMobile(input_shape=(352,352,3), include_top=False, weights=None) if os.path.exists('./NASNet-mobile-no-top.h5'): pass else: urlretrieve('https://github.com/titu1994/Keras-NASNet/releases/download/v1.2/NASNet-mobile-no-top.h5', './NASNet-mobile-no-top.h5') Backbone.load_weights('./NASNet-mobile-no-top.h5') Stages=[Backbone.output, Backbone.get_layer("activation_131").output, Backbone.get_layer("activation_72").output, Backbone.get_layer("activation_13").output] # # # # # # # # # # # # # # # # # # # # # # # # # # # # elif self.Backbone_model == "NASNetLarge": Backbone = NASNetLarge(input_shape=(352,352,3), include_top=False, weights=None) if os.path.exists('./NASNet-large-no-top.h5'): pass else: urlretrieve('https://github.com/titu1994/Keras-NASNet/releases/download/v1.2/NASNet-large-no-top.h5', './NASNet-large-no-top.h5') Backbone.load_weights('./NASNet-large-no-top.h5') Stages=[Backbone.output, Backbone.get_layer("activation_319").output, Backbone.get_layer("activation_260").output, Backbone.get_layer("activation_201").output] # # # # # # # # # # # # # # # # # # # # # # # # # # # # else: raise ValueError("The name of the Backbone_model must be one of the following options: VGG16, ResNet50, NASNetMobile, NASNetLarge") # # # # # # # # # # # # # # # # # # # # # # # # # # # # ExtractedFeatures = [] Number_of_Filters = [192,128,96,64] for Stage,Num in zip(Stages,Number_of_Filters): MAG_output=MAG_Module(Stage, Num) extr=Conv2D(Num, (1, 1), padding='same')(MAG_output) ExtractedFeatures.append(extr) #Extracted features from the Feature Extraction Network ############### Feature Integration Network ################## z = BilinearUpsampling(output_size=(ExtractedFeatures[0]._keras_shape[1]*2, ExtractedFeatures[0]._keras_shape[2]*2))(ExtractedFeatures[0]) for i in range(len(ExtractedFeatures)-1): z = AMI_Module(z, ExtractedFeatures[i+1], ExtractedFeatures[i+1]._keras_shape[-1]) z = BilinearUpsampling(output_size=(z._keras_shape[1]*2,z._keras_shape[2]*2))(z) z = Conv2D(64, (3, 3), padding='same', strides=(1,1))(z) z = BatchNormalization(axis=-1)(z) z = Activation('relu')(z) z = BilinearUpsampling(output_size=(352,352))(z) z = Conv2D(2, (1, 1), padding='same')(z) z = Activation('softmax')(z) self.model = Model(Backbone.input,z) self.model.compile(optimizer=SGD(lr=self.learning_rate, momentum=0.9), loss=Sharpenning_Loss(self.Lambda), metrics=["accuracy"]) if self.show_ModelSummary == True: self.model.summary()