def get_named_model_func(model_name): if model_name == 'Xception': return applications.xception.Xception(weights='imagenet', include_top=False, pooling='avg') if model_name == 'VGG16': return applications.vgg16.VGG16(weights='imagenet', include_top=False, pooling='avg') if model_name == 'VGG19': return applications.vgg19.VGG19(weights='imagenet', include_top=False, pooling='avg') if model_name == 'InceptionV3': return applications.inception_v3.InceptionV3(weights='imagenet', include_top=False, pooling='avg') if model_name == 'InceptionResNetV2': return applications.inception_resnet_v2.InceptionResNetV2(weights='imagenet', include_top=False, pooling='avg') if model_name == 'MobileNet': return applications.mobilenet.MobileNet(weights='imagenet', include_top=False, pooling='avg') if model_name == 'ResNet50': return applications.resnet50.ResNet50(weights='imagenet', include_top=False, pooling='avg') if model_name == 'DenseNet201': return applications.DenseNet201(weights='imagenet', include_top=False, pooling='avg') raise ValueError('Unknown model')
def save_bottlebeck_features(): train_datagen = ImageDataGenerator( rescale=1. / 255, rotation_range=0.2, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True) # train_datagen = ImageDataGenerator(rescale=1. / 255) test_datagen = ImageDataGenerator(rescale=1. / 255) # build the DenseNet201 network model = applications.DenseNet201(include_top=False, weights='imagenet') train_generator = train_datagen.flow_from_directory( train_data_dir, target_size=(img_width, img_height), batch_size=batch_size, class_mode=None, shuffle=False) bottleneck_features_train = model.predict_generator( train_generator, aug_factor * nb_classes * nb_train_samples // batch_size) train_labels = np.tile(np.repeat(np.arange(nb_classes), nb_train_samples), aug_factor) np.savez(bn_train_path, data=bottleneck_features_train, label=train_labels) test_generator = test_datagen.flow_from_directory( validation_data_dir, target_size=(img_width, img_height), batch_size=batch_size, class_mode=None, shuffle=False) bottleneck_features_validation = model.predict_generator( test_generator, nb_classes * nb_validation_samples // batch_size) validation_labels = np.repeat(np.arange(nb_classes), nb_validation_samples) np.savez(bn_validation_path, data=bottleneck_features_validation, label=validation_labels)
def get_imagenet_cnn(architecture, variant, size, alpha, output_layer, include_top=False, weights='imagenet'): from keras import applications, Model if include_top: assert output_layer == '*last*' if size == '*auto*': size = get_image_size(architecture, variant, size) shape = (size, size, 3) if architecture == 'densenet': if variant == '*auto*': variant = 'densenet-121' if variant == 'densenet-121': model = applications.DenseNet121(weights=weights, include_top=include_top, input_shape=shape) elif variant == 'densenet-169': model = applications.DenseNet169(weights=weights, include_top=include_top, input_shape=shape) elif variant == 'densenet-201': model = applications.DenseNet201(weights=weights, include_top=include_top, input_shape=shape) elif architecture == 'inception-resnet-v2': model = applications.InceptionResNetV2(weights=weights, include_top=include_top, input_shape=shape) elif architecture == 'mobilenet': model = applications.MobileNet(weights=weights, include_top=include_top, input_shape=shape, alpha=alpha) elif architecture == 'mobilenet-v2': model = applications.MobileNetV2(weights=weights, include_top=include_top, input_shape=shape, alpha=alpha) elif architecture == 'nasnet': if variant == '*auto*': variant = 'large' if variant == 'large': model = applications.NASNetLarge(weights=weights, include_top=include_top, input_shape=shape) else: model = applications.NASNetMobile(weights=weights, include_top=include_top, input_shape=shape) elif architecture == 'resnet-50': model = applications.ResNet50(weights=weights, include_top=include_top, input_shape=shape) elif architecture == 'vgg-16': model = applications.VGG16(weights=weights, include_top=include_top, input_shape=shape) elif architecture == 'vgg-19': model = applications.VGG19(weights=weights, include_top=include_top, input_shape=shape) elif architecture == 'xception': model = applications.Xception(weights=weights, include_top=include_top, input_shape=shape) elif architecture == 'inception-v3': model = applications.InceptionV3(weights=weights, include_top=include_top, input_shape=shape) if output_layer != '*last*': try: if isinstance(output_layer, int): layer = model.layers[output_layer] else: layer = model.get_layer(output_layer) except Exception: raise VergeMLError('layer not found: {}'.format(output_layer)) model = Model(inputs=model.input, outputs=layer.output) return model
def create_model(existing='', is_twohundred=False, is_halffeatures=True): if len(existing) == 0: print('Loading base model (DenseNet)..') # Encoder Layers if is_twohundred: base_model = applications.DenseNet201(input_shape=(None, None, 3), include_top=False) else: base_model = applications.DenseNet169(input_shape=(None, None, 3), include_top=False) print('Base model loaded.') # Starting point for decoder base_model_output_shape = base_model.layers[-1].output.shape # Layer freezing? for layer in base_model.layers: layer.trainable = True # Starting number of decoder filters if is_halffeatures: decode_filters = int(int(base_model_output_shape[-1]) / 2) else: decode_filters = int(base_model_output_shape[-1]) # Define upsampling layer def upproject(tensor, filters, name, concat_with): up_i = BilinearUpSampling2D((2, 2), name=name + '_upsampling2d')(tensor) up_i = Concatenate(name=name + '_concat')( [up_i, base_model.get_layer(concat_with).output]) # Skip connection up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convA')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convB')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) return up_i # Decoder Layers decoder = Conv2D(filters=decode_filters, kernel_size=1, padding='same', input_shape=base_model_output_shape, name='conv2')(base_model.output) decoder = upproject(decoder, int(decode_filters / 2), 'up1', concat_with='pool3_pool') decoder = upproject(decoder, int(decode_filters / 4), 'up2', concat_with='pool2_pool') decoder = upproject(decoder, int(decode_filters / 8), 'up3', concat_with='pool1') decoder = upproject(decoder, int(decode_filters / 16), 'up4', concat_with='conv1/relu') if False: decoder = upproject(decoder, int(decode_filters / 32), 'up5', concat_with='input_1') # Extract depths (final layer) conv3 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv3')(decoder) # Create the model model = Model(inputs=base_model.input, outputs=conv3) else: # Load model from file if not existing.endswith('.h5'): sys.exit( 'Please provide a correct model file when using [existing] argument.' ) custom_objects = { 'BilinearUpSampling2D': BilinearUpSampling2D, 'depth_loss_function': depth_loss_function } model = load_model(existing, custom_objects=custom_objects) print('\nExisting model loaded.\n') print('Model created.') return model
def train(train_path, val_path, test_path, batch_size=32, epochs=50, network='InceptionResNetV2', data_augmentation=True, mode='finetune', optimizer='Adadelta', fc=1, classes=5, gpu=1): ''' Inputs: train_path: data path for train set (data should be stored like train/DR, train/Normal) val_path: data path for validation set test_path: data path for test set batch_size: data sizes per step epochs: loop counts over whole train set network: { 'InceptionResNetV2': fine-tune mode will train last 2 inception blocks 'DenseNet201': fine-tune mode will train last Dense block 'InceptionV3': fine-tune mode will train last 2 inception blocks 'Xception' 'NASNet' 'MobileNetV2' 'ResNet50': According to https://arxiv.org/pdf/1805.08974.pdf, it is most suitable for transfer learning? } data_augmentation: whether to do data augmentation or not mode: { 'retrain': randomly initialize all layers and retrain the whole model 'finetune': train specified layers 'transfer' train fc layer(s) } optimizer: { 'Adadelta' 'RMSprop' } fc: { 1: only one fc layer at last 2: include two fc layers at last } classes: category counts ''' if mode == 'retrain': include_top = False weights = None pooling = 'avg' else: include_top = False weights = 'imagenet' pooling = 'avg' if network == 'DenseNet201': from keras.applications.densenet import preprocess_input img_width, img_height = 224, 224 base_model = applications.DenseNet201(include_top=include_top, weights=weights, pooling=pooling) # train last Dense Block if mode == 'finetune': trainable = False for layer in base_model.layers: if layer.name == 'conv5_block1_0_bn': trainable = True layer.trainable = trainable if network == 'Xception': from keras.applications.xception import preprocess_input img_width, img_height = 299, 299 base_model = applications.Xception(include_top=include_top, weights=weights, pooling=pooling) if network == 'InceptionV3': from keras.applications.inception_v3 import preprocess_input img_width, img_height = 299, 299 base_model = applications.InceptionV3(include_top=include_top, weights=weights, pooling=pooling) # train top 2 inception blocks if mode == 'finetune': for layer in base_model.layers[:249]: layer.trainable = False for layer in base_model.layers[249:]: #print(layer.name) layer.trainable = True if network == 'InceptionResNetV2': from keras.applications.inception_resnet_v2 import preprocess_input img_width, img_height = 299, 299 base_model = applications.InceptionResNetV2(include_top=include_top, weights=weights, pooling=pooling) # train top 1 inception blocks if mode == 'finetune': trainable = True for layer in base_model.layers: #print(layer.name) if layer.name == 'conv2d_9': trainable = False if layer.name == 'conv2d_201': trainable = True layer.trainable = trainable if network == 'NASNet': from keras.applications.nasnet import preprocess_input img_width, img_height = 331, 331 base_model = applications.NASNetLarge(include_top=include_top, weights=weights, pooling=pooling) if network == 'MoblieNetV2': from keras.applications.mobilenetv2 import preprocess_input img_width, img_height = 224, 224 base_model = applications.MobileNetV2(include_top=include_top, weights=weights, pooling=pooling) if network == 'ResNet50': from keras.applications.resnet50 import preprocess_input img_width, img_height = 224, 224 base_model = applications.ResNet50(include_top=include_top, weights=weights, pooling=pooling) bottleneck = base_model.output if fc == 2: bottleneck = Dense( 512, activation='relu', kernel_regularizer=keras.regularizers.l2(l=0.001))(bottleneck) predictions = Dense( classes, kernel_regularizer=keras.regularizers.l2(l=0.001), activation='softmax', bias_regularizer=keras.regularizers.l2(l=0.001))(bottleneck) model = Model(inputs=base_model.input, outputs=predictions) if mode == 'transfer': # train only the top layers (which were randomly initialized) # freeze all convolutional layers for layer in base_model.layers: layer.trainable = False if mode == 'retrain': # train a complete model for layer in base_model.layers: layer.trainable = True if optimizer == 'Adadelta': opt = optimizers.Adadelta() if optimizer == 'Adam': opt = optimizers.Adam() if optimizer == 'RMSprop': opt = optimizers.RMSprop(lr=0.005, rho=0.9, epsilon=1.0, decay=0.94) if gpu > 1: batch_size *= gpu model = multi_gpu_model(model, gpus=gpu) model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) if data_augmentation: # Initialize the train and test generators with data Augumentation train_datagen = ImageDataGenerator( preprocessing_function=preprocess_input, horizontal_flip=True, fill_mode="nearest", zoom_range=0.3, width_shift_range=0.3, height_shift_range=0.3, rotation_range=30) val_datagen = ImageDataGenerator( preprocessing_function=preprocess_input) else: train_datagen = ImageDataGenerator( preprocessing_function=preprocess_input) val_datagen = ImageDataGenerator( preprocessing_function=preprocess_input) train_generator = train_datagen.flow_from_directory( train_path, target_size=(img_height, img_width), batch_size=batch_size, class_mode="categorical") validation_generator = val_datagen.flow_from_directory( val_path, target_size=(img_height, img_width), class_mode="categorical") test_generator = val_datagen.flow_from_directory(test_path, target_size=(img_height, img_width), class_mode="categorical") checkpoint = ModelCheckpoint("{}_{}_{}.h5".format(network, mode, optimizer), monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) early = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='auto') model.fit_generator(train_generator, epochs=epochs, validation_data=validation_generator, callbacks=[checkpoint, early]) score = model.evaluate_generator(test_generator) print(score)
input_shape=(256, 256, 3)) elif args.base_model == 'xception': base_model = applications.Xception(weights='imagenet', include_top=False, input_shape=(256, 256, 3)) elif args.base_model == 'densenet121': base_model = applications.DenseNet121(weights='imagenet', include_top=False, input_shape=(256, 256, 3)) elif args.base_model == 'densenet169': base_model = applications.DenseNet169(weights='imagenet', include_top=False, input_shape=(256, 256, 3)) elif args.base_model == 'densenet201': base_model = applications.DenseNet201(weights='imagenet', include_top=False, input_shape=(256, 256, 3)) elif args.base_model == 'nasnetmobile': base_model = applications.NASNetMobile(weights='imagenet', include_top=False, input_shape=(256, 256, 3)) elif args.base_model == 'nasnetlarge': base_model = applications.NASNetLarge(weights='imagenet', include_top=False, input_shape=(256, 256, 3)) else: raise ValueError( "Model you entered is not present in the model zoo thats offered") if args.bottleneck_tensorname is None: # Taking the last tensor (Just before the softmax layer)
def create_two_branch_model(existing='', is_twohundred=False, is_halffeatures=True, channels=4): if len(existing) == 0: print('Loading base model (DenseNet)..') def crop(dimension, start, end): # Crops (or slices) a Tensor on a given dimension from start to end # example : to crop tensor x[:, :, 5:10] # call slice(2, 5, 10) as you want to crop on the second dimension def func(x): if dimension == 0: return x[start: end] if dimension == 1: return x[:, start: end] if dimension == 2: return x[:, :, start: end] if dimension == 3: return x[:, :, :, start: end] if dimension == 4: return x[:, :, :, :, start: end] return Lambda(func) input_rgbd = Input(shape=(None, None, channels), name='input_rgbd') input_rgb = crop(3, 0, 3)(input_rgbd) input_sparse = crop(3, 3, channels)(input_rgbd) #base_model_sz_input = Conv2D(3, (3,3), padding='same')(input_sparse) # Encoder Layers if is_twohundred: base_model = applications.DenseNet201(input_shape=(None, None, 3), include_top=False, weights='imagenet', input_tensor=input_rgb) #base_model_sz = applications.DenseNet201(input_shape=(None, None, 3), include_top=False, weights='imagenet', input_tensor=concatenate([input_sparse, input_sparse, input_sparse], axis=-1)) base_model_sz = applications.DenseNet201(input_shape=(None, None, channels-3), include_top=False, weights=None, input_tensor=input_sparse) else: base_model = applications.DenseNet169(input_shape=(None, None, 3), include_top=False, weights='imagenet', input_tensor=input_rgb) #base_model_sz = applications.DenseNet169(input_shape=(None, None, 3), include_top=False, weights='imagenet', input_tensor=concatenate([input_sparse, input_sparse, input_sparse], axis=-1)) base_model_sz = applications.DenseNet121(input_shape=(None, None, channels-3), include_top=False, weights=None, input_tensor=input_sparse) pretrained = applications.DenseNet121(input_shape=(None, None, 3), include_top=False, weights='imagenet') for layer in pretrained.layers: print(layer.name) if layer.get_weights() != []: # Skip input, pooling and no weights layers target_layer = base_model_sz.get_layer(name=layer.name) if layer.name != 'conv1/conv': # Initialize imagenet weights in all layers except the first conv1 layer where the channels do not match target_layer.set_weights(layer.get_weights()) print('Base model loaded.') # Layer freezing? for layer in base_model.layers: layer.trainable = True for layer in base_model_sz.layers: layer.trainable = True layer.name = layer.name + str("_sz") # Starting point for decoder encoder_output = concatenate([base_model.output, base_model_sz.output], axis=-1) base_model_output_shape = encoder_output.shape #base_model_output_shape = base_model.layers[-1].output.shape #base_model_output_shape_sz = base_model_sz.layers[-1].output.shape # Starting number of decoder filters if is_halffeatures: decode_filters = int(int(base_model_output_shape[-1])/2) else: decode_filters = int(base_model_output_shape[-1]) # Define upsampling layer def upproject(tensor, filters, name, concat_with): up_i = BilinearUpSampling2D((2, 2), name=name+'_upsampling2d')(tensor) up_i = Concatenate(name=name+'_concat')([up_i, base_model.get_layer(concat_with).output, base_model_sz.get_layer(concat_with+str("_sz")).output]) # Skip connection up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name+'_convA')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name+'_convB')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) return up_i # Decoder Layers decoder = Conv2D(filters=decode_filters, kernel_size=1, padding='same', input_shape=base_model_output_shape, name='conv2')(encoder_output) decoder = upproject(decoder, int(decode_filters/2), 'up1', concat_with='pool3_pool') decoder = upproject(decoder, int(decode_filters/4), 'up2', concat_with='pool2_pool') decoder = upproject(decoder, int(decode_filters/8), 'up3', concat_with='pool1') decoder = upproject(decoder, int(decode_filters/16), 'up4', concat_with='conv1/relu') if False: decoder = upproject(decoder, int(decode_filters/32), 'up5', concat_with='input_1') # Extract depths (final layer) conv3 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv3')(decoder) # Create the model model = Model(inputs=input_rgbd, outputs=conv3) else: # Load model from file if not existing.endswith('.h5'): sys.exit('Please provide a correct model file when using [existing] argument.') custom_objects = {'BilinearUpSampling2D': BilinearUpSampling2D, 'depth_loss_function': depth_loss_function} model = load_model(existing, custom_objects=custom_objects, compile=False) print('\nExisting model loaded.\n') print('Model created.') return model
def mySpatialModel(model_name, spatial_size, nb_classes, channels, weights_path=None): input_tensor = Input(shape=(channels, spatial_size, spatial_size)) input_shape = (channels, spatial_size, spatial_size) base_model = None predictions = None data_dim = 1024 if model_name == 'ResNet50': input_tensor = Input(shape=(spatial_size, spatial_size, channels)) input_shape = (spatial_size, spatial_size, channels) base_model = kerasApp.ResNet50(include_top=False, input_tensor=input_tensor, input_shape=input_shape, weights=weights_path, classes=nb_classes, pooling=None) x = base_model.output # 添加自己的全链接分类层 method 1 #x = Flatten()(x) #predictions = Dense(nb_classes, activation='softmax')(x) #method 2 x = GlobalAveragePooling2D()(x) x = Dense(1024, activation='relu')(x) predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'VGG16': input_tensor = Input(shape=(spatial_size, spatial_size, channels)) input_shape = (spatial_size, spatial_size, channels) base_model = kerasApp.VGG16(include_top=False, input_tensor=input_tensor, input_shape=input_shape, weights=weights_path, classes=nb_classes, pooling=None) x = base_model.output x = GlobalAveragePooling2D()( x) # add a global spatial average pooling layer x = Dense(1024, activation='relu')(x) # let's add a fully-connected layer predictions = Dense(nb_classes, activation='softmax')(x) # and a logistic layer model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'VGG19': base_model = kerasApp.VGG19(include_top=False, input_tensor=input_tensor, input_shape=input_shape, weights=weights_path, classes=2, pooling=None) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) x = Dense(1024, activation='relu')(x) predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'InceptionV3': input_tensor = Input(shape=(spatial_size, spatial_size, channels)) input_shape = (spatial_size, spatial_size, channels) base_model = kerasApp.InceptionV3(weights=weights_path, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) x = Dense(1024, activation='relu')(x) predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'InceptionResNetV2': input_tensor = Input(shape=(spatial_size, spatial_size, channels)) input_shape = ( spatial_size, spatial_size, channels, ) base_model = kerasApp.InceptionResNetV2(weights=weights_path, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) data_dim = 1536 predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'Xception': input_shape_xception = (spatial_size, spatial_size, channels) base_model = kerasApp.Xception(weights=weights_path, include_top=False, pooling="avg", input_shape=input_shape_xception, classes=nb_classes) x = base_model.output predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'DenseNet121': base_model = kerasApp.DenseNet121(weights=weights_path, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'DenseNet169': base_model = kerasApp.DenseNet169(weights=weights_path, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'DenseNet201': base_model = kerasApp.DenseNet201(weights=weights_path, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) elif model_name == 'MobileNet': base_model = kerasApp.MobileNet(weights=weights_path, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) x = Dense(1024, activation='relu')(x) x = Dense(1024, activation='relu')(x) x = Dense(512, activation='relu')(x) data_dim = 512 predictions = Dense(nb_classes, activation='softmax')(x) model = Model(inputs=base_model.input, outputs=predictions) else: print("this model--[" + model_name + "]-- doesnt exist!") # 冻结base_model所有层,这样就可以正确获得bottleneck特征 for layer in base_model.layers: layer.trainable = True # 训练模型 model = Model(inputs=base_model.input, outputs=predictions) print('-------------当前base_model模型[' + model_name + "]-------------------\n") print('base_model层数目:' + str(len(base_model.layers))) print('model模型层数目:' + str(len(model.layers))) featureLayer = model.layers[len(model.layers) - 2] print(featureLayer.output_shape) print("data_dim:" + str(featureLayer.output_shape[1])) print("---------------------------------------------\n") #sgd = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True) # 绘制模型 #if plot_model: # plot_model(model, to_file=model_name+'.png', show_shapes=True) return model
def create_model(is_twohundred=False, is_halffeatures=True): print('Loading base model (DenseNet)..') # Encoder Layers if is_twohundred: base_model = applications.DenseNet201(input_shape=(480, 640, 3), include_top=False) else: base_model = applications.DenseNet169(input_shape=(480, 640, 3), include_top=False) print('Base model loaded.') # Starting point for decoder base_model_output_shape = base_model.layers[-1].output.shape # Layer freezing? for layer in base_model.layers: layer.trainable = True # Starting number of decoder filters if is_halffeatures: decode_filters = int(int(base_model_output_shape[-1]) / 2) else: decode_filters = int(base_model_output_shape[-1]) def expend_as(tensor, rep): return layers.Lambda(lambda x, repnum: K.repeat_elements(x, repnum, axis=3), arguments={'repnum': rep})(tensor) # Define upsampling layer def upproject(tensor, filters, name, concat_with): up_i = BilinearUpSampling2D((2, 2), name=name + '_upsampling2d')(tensor) up_i = Concatenate(name=name + '_concat')( [up_i, concat_with]) # Skip connection up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convA')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convB')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) return up_i def global_non_local(X,cc): h, w , c = list(X.shape)[1], list(X.shape)[2], list(X.shape)[3] c=cc theta = Conv2D(c, kernel_size=(1,1), padding='same')(X) theta_rsh = Reshape((h*w, c))(theta) phi = Conv2D(c, kernel_size=(1,1), padding='same')(X) phi_rsh = Reshape((c, h*w))(phi) g = Conv2D(c, kernel_size=(1,1), padding='same')(X) g_rsh = Reshape((h*w, c))(g) theta_phi = tf.matmul(theta_rsh, phi_rsh) theta_phi = tf.keras.layers.Softmax()(theta_phi) theta_phi_g = tf.matmul(theta_phi, g_rsh) theta_phi_g = Reshape((h, w, c))(theta_phi_g) theta_phi_g = Conv2D(c*2, kernel_size=(1,1), padding='same')(theta_phi_g) out = Add()([theta_phi_g, X]) return out def gating_signal(input, out_size, batch_norm=False): """ resize the down layer feature map into the same dimension as the up layer feature map using 1x1 conv :param input: down-dim feature map :param out_size:output channel number :return: the gating feature map with the same dimension of the up layer feature map """ x = keras.layers.Conv2D(out_size, (1, 1), padding='same')(input) if batch_norm: x = keras.layers.BatchNormalization()(x) x = keras.layers.Activation('relu')(x) return x def attention_block(x, gating, inter_shape): shape_x = K.int_shape(x) shape_g = K.int_shape(gating) theta_x = layers.Conv2D(inter_shape, (2, 2), strides=(2, 2), padding='same')(x) # 16 shape_theta_x = K.int_shape(theta_x) phi_g = layers.Conv2D(inter_shape, (1, 1), padding='same')(gating) upsample_g = layers.Conv2DTranspose(inter_shape, (3, 3), strides=(shape_theta_x[1] // shape_g[1], shape_theta_x[2] // shape_g[2]), padding='same')(phi_g) # 16 concat_xg = layers.add([upsample_g, theta_x]) act_xg = layers.Activation('relu')(concat_xg) psi = layers.Conv2D(1, (1, 1), padding='same')(act_xg) sigmoid_xg = layers.Activation('sigmoid')(psi) shape_sigmoid = K.int_shape(sigmoid_xg) upsample_psi = layers.UpSampling2D(size=(shape_x[1] // shape_sigmoid[1], shape_x[2] // shape_sigmoid[2]))( sigmoid_xg) # 32 upsample_psi = expend_as(upsample_psi, shape_x[3]) y = layers.multiply([upsample_psi, x]) result = layers.Conv2D(shape_x[3], (1, 1), padding='same')(y) result_bn = layers.BatchNormalization()(result) return result_bn print('decode_filters=',decode_filters) non_local=global_non_local(base_model.output,decode_filters) # Decoder Layers decoder_1 = Conv2D(filters=decode_filters, kernel_size=1, padding='same', input_shape=base_model_output_shape, name='conv2')(non_local) gating_2=gating_signal(decoder_1,int(decode_filters/2),False) att2=attention_block(base_model.get_layer('pool3_pool').output,gating_2,int(decode_filters/2)) decoder_2 = upproject(decoder_1, int(decode_filters / 2), 'up1', concat_with=att2) gating_3 = gating_signal(decoder_2, int(decode_filters / 4), False) att3 = attention_block(base_model.get_layer('pool2_pool').output, gating_3, int(decode_filters / 4)) decoder_3 = upproject(decoder_2, int(decode_filters / 4), 'up2', concat_with=att3) # conv3 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv3_')(decoder) gating_4 = gating_signal(decoder_3,int(decode_filters/8),False) att4 = attention_block(base_model.get_layer('pool1').output,gating_4,int(decode_filters/8)) decoder_4 = upproject(decoder_3, int(decode_filters / 8), 'up3', concat_with=att4) # conv2 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv2_')(decoder) gating_5 = gating_signal(decoder_4,int(decode_filters/16),False) att5 = attention_block(base_model.get_layer('conv1/relu').output,gating_5,int(decode_filters/16)) decoder_5 = upproject(decoder_4, int(decode_filters / 16), 'up4', concat_with=att5) # conv1 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv1_')(decoder) gating_6 = gating_signal(decoder_5,int(decode_filters/32),False) att6=attention_block(base_model.get_layer('input_1').output,gating_6,int(decode_filters/32)) decoder_6 = upproject(decoder_5, int(decode_filters / 32), 'up5', concat_with=att6) conv0 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv0_')(decoder_6) predictions_raw_0 = conv0 * 1000 # predictions_raw_1 = conv1 * 1000 # predictions_raw_2 = conv2 * 1000 # predictions_raw_3 = conv3 * 1000 predictions_0 = tf.clip_by_value(predictions_raw_0, 1.0, 65535.0) # predictions_1 = tf.clip_by_value(predictions_raw_1, 1.0, 65535.0) # predictions_2 = tf.clip_by_value(predictions_raw_2, 1.0, 65535.0) # predictions_3 = tf.clip_by_value(predictions_raw_3, 1.0, 65535.0) final_outputs_0 = tf.cast(tf.clip_by_value(predictions_raw_0, 0, 65535.0), tf.int16) eval_outputs__0 = tf.clip_by_value(predictions_raw_0, 0.1, 65535.0) # final_outputs_1 = tf.cast(tf.clip_by_value(predictions_raw_1, 0, 65535.0), tf.int16) # final_outputs_2 = tf.cast(tf.clip_by_value(predictions_raw_2, 0, 65535.0), tf.int16) # final_outputs_3 = tf.cast(tf.clip_by_value(predictions_raw_3, 0, 65535.0), tf.int16) # Create the model model = Model(inputs=base_model.input, outputs= [predictions_0, final_outputs_0, eval_outputs__0]) model.summary() print('Model created.') return model
def mySpatialModelChannelTest(model_name,spatial_size, nb_classes, channels, channel_first=True, weights_path=None, lr=0.005, decay=1e-6, momentum=0.9,plot_model=True): input_tensor = Input(shape=(channels, spatial_size, spatial_size)) input_shape = (channels, spatial_size, spatial_size) base_model=None predictions=None data_dim=1024 base_model = kerasApp.ResNet50(include_top=False, input_tensor=input_tensor, input_shape=input_shape, weights=None, classes=nb_classes, pooling=None) x = base_model.output x = GlobalAveragePooling2D()(x) x = Dense(1024, activation='relu')(x) predictions = Dense(nb_classes, activation='softmax')(x) # 训练模型 model = Model(inputs=base_model.input, outputs=predictions) print_shape(model,model_name) base_model = kerasApp.VGG16(include_top=False, input_tensor=input_tensor, input_shape=input_shape, weights=None, classes=nb_classes, pooling=None) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) # add a global spatial average pooling layer x = Dense(1024, activation='relu')(x) # let's add a fully-connected layer predictions = Dense(nb_classes, activation='softmax')(x) # 训练模型 model = Model(inputs=base_model.input, outputs=predictions) print_shape(model, model_name) base_model = kerasApp.VGG19(include_top=False, input_tensor=input_tensor, input_shape=input_shape, weights=None, classes=2, pooling='avg') print_shape(base_model, model_name) base_model = kerasApp.InceptionV3(weights=None, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) print_shape(base_model, model_name) base_model = kerasApp.InceptionResNetV2(weights=None, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) x = base_model.output # 添加自己的全链接分类层 x = GlobalAveragePooling2D()(x) predictions = Dense(nb_classes, activation='softmax')(x) # 训练模型 model = Model(inputs=base_model.input, outputs=predictions) print_shape(model, model_name) #channel last input_tensor_Xception = Input(shape=( spatial_size, spatial_size,channels)) input_shape__Xception = (spatial_size, spatial_size,channels) base_model = kerasApp.Xception(weights=None, include_top=False, pooling=None, input_shape=input_shape__Xception, classes=nb_classes) print_shape(base_model, model_name) base_model = kerasApp.DenseNet121(weights=None, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) print_shape(base_model, model_name) base_model = kerasApp.DenseNet169(weights=None, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) print_shape(base_model, model_name) base_model = kerasApp.DenseNet201(weights=None, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes) print_shape(base_model, model_name) input_shape = (channels, spatial_size, spatial_size) base_model = kerasApp.MobileNet(weights=None, include_top=False, pooling=None, input_shape=input_shape, classes=nb_classes)
def create_model(is_twohundred=False, is_halffeatures=True): print('Loading base model (DenseNet)..') # Encoder Layers if is_twohundred: base_model = applications.DenseNet201(input_shape=(480, 640, 3), include_top=False) else: #base_model = applications.DenseNet169(input_shape=(480, 640, 3), include_top=False) base_model = efn.EfficientNetB1(weights='imagenet',include_top=False,input_shape=((480, 640, 3))) print('Base model loaded.') # Starting point for decoder base_model_output_shape = base_model.layers[-1].output.shape # Layer freezing? for layer in base_model.layers: layer.trainable = True # Starting number of decoder filters if is_halffeatures: decode_filters = int(int(base_model_output_shape[-1]) / 2) else: decode_filters = int(base_model_output_shape[-1]) base_model.summary() # Define upsampling layer def upproject(tensor, filters, name, concat_with): up_i = BilinearUpSampling2D((2, 2), name=name + '_upsampling2d')(tensor) up_i = Concatenate(name=name + '_concat')( [up_i, base_model.get_layer(concat_with).output]) # Skip connection up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convA')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convB')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) return up_i def global_non_local(X,cc): h, w , c = list(X.shape)[1], list(X.shape)[2], list(X.shape)[3] c=cc theta = Conv2D(c, kernel_size=(1,1), padding='same')(X) theta_rsh = Reshape((h*w, c))(theta) phi = Conv2D(c, kernel_size=(1,1), padding='same')(X) phi_rsh = Reshape((c, h*w))(phi) g = Conv2D(c, kernel_size=(1,1), padding='same')(X) g_rsh = Reshape((h*w, c))(g) theta_phi = tf.matmul(theta_rsh, phi_rsh) theta_phi = tf.keras.layers.Softmax()(theta_phi) theta_phi_g = tf.matmul(theta_phi, g_rsh) theta_phi_g = Reshape((h, w, c))(theta_phi_g) theta_phi_g = Conv2D(c*2, kernel_size=(1,1), padding='same')(theta_phi_g) out = Add()([theta_phi_g, X]) return out non_local=global_non_local(base_model.output,decode_filters) # Decoder Layers decoder = Conv2D(filters=decode_filters, kernel_size=1, padding='same', input_shape=base_model_output_shape, name='conv2')(non_local) decoder = upproject(decoder, int(decode_filters / 2), 'up1', concat_with='block5d_add') decoder = upproject(decoder, int(decode_filters / 4), 'up2', concat_with='block3c_add') # conv3 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv3_')(decoder) decoder = upproject(decoder, int(decode_filters / 8), 'up3', concat_with='block2c_add') # conv2 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv2_')(decoder) decoder = upproject(decoder, int(decode_filters / 16), 'up4', concat_with='block1b_add') # conv1 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv1_')(decoder) decoder = upproject(decoder, int(decode_filters / 32), 'up5', concat_with='input_1') conv0 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv0_')(decoder) ''' decoder = Conv2D(filters=decode_filters, kernel_size=1, padding='same', input_shape=base_model_output_shape, name='conv2')(base_model.output) decoder = upproject(decoder, int(decode_filters / 2), 'up1', concat_with='pool3_pool') decoder = upproject(decoder, int(decode_filters / 4), 'up2', concat_with='pool2_pool') decoder = upproject(decoder, int(decode_filters / 8), 'up3', concat_with='pool1') decoder = upproject(decoder, int(decode_filters / 16), 'up4', concat_with='conv1/relu') decoder = upproject(decoder, int(decode_filters / 32), 'up5', concat_with='input_1') conv0 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv0_')(decoder) ''' predictions_raw_0 = conv0 * 1000 # predictions_raw_1 = conv1 * 1000 # predictions_raw_2 = conv2 * 1000 # predictions_raw_3 = conv3 * 1000 predictions_0 = tf.clip_by_value(predictions_raw_0, 1.0, 65535.0) # predictions_1 = tf.clip_by_value(predictions_raw_1, 1.0, 65535.0) # predictions_2 = tf.clip_by_value(predictions_raw_2, 1.0, 65535.0) # predictions_3 = tf.clip_by_value(predictions_raw_3, 1.0, 65535.0) final_outputs_0 = tf.cast(tf.clip_by_value(predictions_raw_0, 0, 65535.0), tf.int16) eval_outputs__0 = tf.clip_by_value(predictions_raw_0, 0.1, 65535.0) # final_outputs_1 = tf.cast(tf.clip_by_value(predictions_raw_1, 0, 65535.0), tf.int16) # final_outputs_2 = tf.cast(tf.clip_by_value(predictions_raw_2, 0, 65535.0), tf.int16) # final_outputs_3 = tf.cast(tf.clip_by_value(predictions_raw_3, 0, 65535.0), tf.int16) # Create the model model = Model(inputs=base_model.input, outputs= [predictions_0, final_outputs_0, eval_outputs__0]) model.summary() #print(base_model.output) #print(base_model.layers[-1].output) print('Model created.') return model #a=create_model()
def create_model(args, is_twohundred=False, is_halffeatures=True): print('Loading base model (DenseNet)..') # Encoder Layers if is_twohundred: base_model = applications.DenseNet201(input_shape=(480, 640, 3), include_top=False) else: base_model = applications.DenseNet169(input_shape=(480, 640, 3), include_top=False) print('Base model loaded.') # Starting point for decoder base_model_output_shape = base_model.layers[-1].output.shape # Layer freezing? for layer in base_model.layers: layer.trainable = True # Starting number of decoder filters if is_halffeatures: decode_filters = int(int(base_model_output_shape[-1]) / 2) else: decode_filters = int(base_model_output_shape[-1]) # Define upsampling layer def upproject(tensor, filters, name, concat_with): up_i = BilinearUpSampling2D((2, 2), name=name + '_upsampling2d')(tensor) up_i = Concatenate(name=name + '_concat')( [up_i, base_model.get_layer(concat_with).output]) # Skip connection up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convA')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) up_i = Conv2D(filters=filters, kernel_size=3, strides=1, padding='same', name=name + '_convB')(up_i) up_i = LeakyReLU(alpha=0.2)(up_i) return up_i # Decoder Layers decoder = Conv2D(filters=decode_filters, kernel_size=1, padding='same', input_shape=base_model_output_shape, name='conv2')(base_model.output) decoder = upproject(decoder, int(decode_filters / 2), 'up1', concat_with='pool3_pool') decoder = upproject(decoder, int(decode_filters / 4), 'up2', concat_with='pool2_pool') # conv3 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv3_')(decoder) decoder = upproject(decoder, int(decode_filters / 8), 'up3', concat_with='pool1') # conv2 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv2_')(decoder) decoder = upproject(decoder, int(decode_filters / 16), 'up4', concat_with='conv1/relu') # conv1 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv1_')(decoder) decoder = upproject(decoder, int(decode_filters / 32), 'up5', concat_with='input_1') conv0 = Conv2D(filters=1, kernel_size=3, strides=1, padding='same', name='conv0_')(decoder) predictions_raw_0 = conv0 * 1000 # predictions_raw_1 = conv1 * 1000 # predictions_raw_2 = conv2 * 1000 # predictions_raw_3 = conv3 * 1000 predictions_0 = tf.clip_by_value(predictions_raw_0, 1.0, 65535.0) # predictions_1 = tf.clip_by_value(predictions_raw_1, 1.0, 65535.0) # predictions_2 = tf.clip_by_value(predictions_raw_2, 1.0, 65535.0) # predictions_3 = tf.clip_by_value(predictions_raw_3, 1.0, 65535.0) final_outputs_0 = tf.cast(tf.clip_by_value(predictions_raw_0, 0, 65535.0), tf.int16) eval_outputs__0 = tf.clip_by_value(predictions_raw_0, 0.1, 65535.0) # final_outputs_1 = tf.cast(tf.clip_by_value(predictions_raw_1, 0, 65535.0), tf.int16) # final_outputs_2 = tf.cast(tf.clip_by_value(predictions_raw_2, 0, 65535.0), tf.int16) # final_outputs_3 = tf.cast(tf.clip_by_value(predictions_raw_3, 0, 65535.0), tf.int16) # Create the model model = Model(inputs=base_model.input, outputs= [predictions_0, final_outputs_0, eval_outputs__0]) model.summary() print('Model created.') return model