def NASNet_ensemble_FCN(input_image, weights=None, fine_tune=False):
input_tensor = Input(shape=(input_image))
model = NASNetMobile(input_shape=(224, 224, 3),
input_tensor=input_tensor,
include_top=False,
weights=weights)
#print model.summary()
#return
#For fine-tuning
if fine_tune:
nasnet_layers = model.layers
for layer in nasnet_layers:
print layer
layer.trainable = False
stem_2 = model.get_layer(name='reduction_concat_stem_2').output
reduce_4 = model.get_layer(name='reduction_concat_reduce_4').output
normal_12 = model.get_layer(name='normal_concat_12').output
#fcn_8s
conv_normal_12 = Conv2D(filters=6, kernel_size=(1, 1))(normal_12)
upscore_normal_12 = Conv2DTranspose(filters=6,
kernel_size=(4, 4),
strides=(2, 2),
padding='same')(conv_normal_12)
conv_reduce_4 = Conv2D(filters=6, kernel_size=(1, 1))(reduce_4)
fuse_4 = Add()([conv_reduce_4, upscore_normal_12])
upscore_fuse_4 = Conv2DTranspose(filters=6,
kernel_size=(4, 4),
strides=(2, 2),
padding='same')(fuse_4)
conv_stem_2 = Conv2D(filters=6, kernel_size=(1, 1))(stem_2)
fuse_2 = Add()([conv_stem_2, upscore_fuse_4])
output_8 = Conv2DTranspose(filters=6,
kernel_size=(16, 16),
strides=(8, 8),
padding='same')(fuse_2)
output_16 = Conv2DTranspose(filters=6,
kernel_size=(32, 32),
strides=(16, 16),
padding='same')(fuse_4)
#fcn_32s
output_32 = Conv2DTranspose(filters=6,
kernel_size=(64, 64),
strides=(32, 32),
padding='same')(conv_normal_12)
model = Model(inputs=input_tensor,
outputs=[output_8, output_16, output_32])
print model.summary()
return model
def save_model11(new_model_path, conv_model_path):
model = NASNetMobile(
input_shape=(img_width, img_height, 3),
include_top=False,
weights=None
)
if pretrained:
model = NASNetMobile(
input_shape=(img_width, img_height, 3),
include_top=False,
weights='imagenet'
)
model.summary()
transfer_layer = model.get_layer('?')
conv_model = Model(inputs=model.input,
outputs=transfer_layer.output)
new_model = Sequential()
new_model.add(conv_model)
new_model.add(GlobalAveragePooling2D())
if num_fc_layers>=1:
new_model.add(Dense(num_fc_neurons, activation='relu'))
if num_fc_layers>=2:
new_model.add(Dropout(dropout))
new_model.add(Dense(num_fc_neurons, activation='relu'))
if num_fc_layers>=3:
new_model.add(Dropout(dropout))
new_model.add(Dense(num_fc_neurons, activation='relu'))
new_model.add(Dense(num_classes, activation='softmax'))
print(new_model.summary())
new_model.save(new_model_path)
conv_model.save(conv_model_path)
return
def __init__(self, cnnModel, weights=None):
K.clear_session()
self.cnnModel = cnnModel
self.weights = weights # so we can check elsewhere which model
if weights is None:
if cnnModel == 'InceptionV3':
# Get model with pretrained weights.
base_model = InceptionV3(weights='imagenet', include_top=True)
# We'll extract features at the final pool layer.
self.model = Model(
inputs=base_model.input,
outputs=base_model.get_layer('avg_pool').output)
self.target_size = (299, 299)
self.preprocess_input = inception_v3_preprocessor
elif cnnModel == 'VGG19':
# Get model with pretrained weights.
base_model = VGG19(weights='imagenet', include_top=True)
# We'll extract features at the final pool layer.
self.model = Model(inputs=base_model.input,
outputs=base_model.get_layer('fc2').output)
self.target_size = (224, 224)
self.preprocess_input = vgg19_preprocessor
elif cnnModel == 'InceptionResNetV2':
# Get model with pretrained weights.
base_model = InceptionResNetV2(weights='imagenet',
include_top=True)
# We'll extract features at the final pool layer.
self.model = Model(
inputs=base_model.input,
outputs=base_model.get_layer('avg_pool').output)
self.target_size = (299, 299)
self.preprocess_input = inception_resnet_v2_preprocessor
elif cnnModel == 'NASNetMobile':
# Get model with pretrained weights.
base_model = NASNetMobile(weights='imagenet', include_top=True)
# We'll extract features at the final pool layer.
self.model = Model(inputs=base_model.input,
outputs=base_model.get_layer(
'global_average_pooling2d_1').output)
self.target_size = (224, 224)
self.preprocess_input = nasnet_preprocessor
elif cnnModel == 'NASNetMobileCropTo11':
# Get model with pretrained weights.
base_model = NASNetMobile(weights='imagenet', include_top=True)
# We'll extract features at the final pool layer.
interModel = Model(
inputs=base_model.input,
outputs=base_model.get_layer('activation_188').output)
self.model = Sequential()
self.model.add(interModel)
self.model.add(Cropping2D(cropping=((3, 3), (3, 3))))
self.model.add(Flatten())
self.target_size = (224, 224)
self.preprocess_input = nasnet_preprocessor
elif cnnModel == 'NASNetMobileCropTo33':
base_model = NASNetMobile(weights='imagenet', include_top=True)
# We'll extract features at the final pool layer.
interModel = Model(
inputs=base_model.get_layer(index=0).input,
outputs=base_model.get_layer(index=-3).output)
self.model = Sequential()
self.model.add(interModel)
self.model.add(Cropping2D(cropping=((2, 2), (2, 2))))
self.model.add(Flatten())
self.target_size = (224, 224)
self.preprocess_input = nasnet_preprocessor
elif cnnModel == 'NASNetMobileOLD':
# Get model with pretrained weights.
base_model = NASNetMobile(weights='imagenet',
include_top=False,
pooling='none')
self.model = base_model
self.target_size = (224, 224)
self.preprocess_input = nasnet_preprocessor
elif cnnModel == 'NASNetLarge':
# Get model with pretrained weights.
base_model = NASNetLarge(weights='imagenet', include_top=True)
# We'll extract features at the final pool layer.
self.model = Model(inputs=base_model.input,
outputs=base_model.get_layer(
'global_average_pooling2d_1').output)
self.target_size = (331, 331)
self.preprocess_input = nasnet_preprocessor
elif cnnModel == 'cifar10vgg':
base_model = cifar10vgg(False)
interModel = Model(
inputs=base_model.model.input,
# outputs=base_model.model.get_layer('flatten_1').output
outputs=base_model.model.get_layer(
'max_pooling2d_3').output
# outputs=base_model.model.get_layer('dropout_10').output
# outputs=base_model.model.get_layer('max_pooling2d_3').output
)
self.model = Sequential()
self.model.add(interModel)
self.model.add(Flatten())
self.target_size = (32, 32)
else:
# Load the model first.
self.model = load_model(weights)
# Then remove the top so we get features not predictions.
# From: https://github.com/fchollet/keras/issues/2371
self.model.layers.pop()
self.model.layers.pop() # two pops to get to pool layer
self.model.outputs = [self.model.layers[-1].output]
self.model.output_layers = [self.model.layers[-1]]
self.model.layers[-1].outbound_nodes = []
class FeatureExtraction():
"""
Feature Extraction class for extracting features from image data
through pre-trained model and saving features and labels to
user defined path.
"""
def __init__(self, model="mobilenet", weights="imagenet", include_top=True):
if model_name == "vgg16":
self.base_model = VGG16(weights=weights)
self.model = Model(input=self.base_model.input, output=self.base_model.get_layer('fc1').output)
self.image_size = (224, 224)
elif model_name == "vgg19":
self.base_model = VGG19(weights=weights)
self.model = Model(input=self.base_model.input, output=self.base_model.get_layer('fc1').output)
self.image_size = (224, 224)
elif model_name == "resnet50":
self.base_model = ResNet50(weights=weights)
self.base_model.summary()
self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('fc1000').output)
self.image_size = (224, 224)
elif model_name == "inceptionv3":
self.base_model = InceptionV3(include_top=include_top, weights=weights, input_tensor=Input(shape=(299,299,3)))
self.model = Model(input=self.base_model.input, output=self.base_model.get_layer('custom').output)
self.image_size = (299, 299)
elif model_name == "inceptionresnetv2":
self.base_model = InceptionResNetV2(include_top=include_top, weights=weights, input_tensor=Input(shape=(299,299,3)))
self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('custom').output)
self.image_size = (299, 299)
elif model_name == "mobilenet":
self.base_model = MobileNet(include_top=include_top, weights=weights, input_tensor=Input(shape=(224,224,3)), input_shape=(224,224,3))
self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('conv_pw_13_relu').output)
self.image_size = (224, 224)
elif model_name == "mobilenetv2":
self.base_model = MobileNeV2(include_top=include_top, weights=weights, input_tensor=Input(shape=(224,224,3)), input_shape=(224,224,3))
self.base_model.summary()
self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('conv_pw_13_relu').output)
self.image_size = (224, 224)
elif model_name == "xception":
self.base_model = Xception(weights=weights, , input_tensor=Input(shape=(299,299,3)))
self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('avg_pool').output)
self.image_size = (299, 299)
elif model_name == "densenet":
self.base_model = DenseNet121(include_top=include_top, weights=weights, input_tensor=Input(shape=(224,224,3)), input_shape=(224,224,3))
self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('avg_pool').output)
self.image_size = (224, 224)
elif model_name == "xception":
self.base_model = NASNetMobile(include_top=include_top, weights=weights, input_tensor=Input(shape=(224,224,3)), input_shape=(224,224,3))
self.model = Model(inputs=self.base_model.input, outputs=self.base_model.get_layer('avg_pool').output)
self.image_size = (224, 224)
else:
self.base_model = None
self.features = []
self.labels = []
def list_directory(self, path=None):
train_labels = os.listdir(path)
return
def label_encoder(self, train_labels=None):
le = LabelEncoder()
le.fit(train_labels)
self.le_labels = le.transform(self.labels)
return le, self.le_labels
def extract_feature(self, features_path='', train_path=path, train_labels=train_labels):
# loop over all the labels in the folder
count = 0
for i, label in enumerate(train_labels):
cur_path = train_path + "/" + label
for image_path in glob.glob(cur_path + "/*.png"):
flat = process_image(image_path)
features.append(flat)
labels.append(label)
count += 1
print ("[INFO] processed - " + str(count))
print ("[STATUS] training labels: {}".format(le_labels))
print ("[STATUS] training labels shape: {}".format(le_labels.shape))
def process_image(self, image_path=path):
# extract features from images
img = image.load_img(image_path, target_size=self.image_size)
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = preprocess_input(img)
feature = model.predict(img)
flat = feature.flatten()
return flat
def NASNetMobile_enc_dec(H=256,
W=256,
weights='imagenet',
ninchannel=3,
noutchannel=1,
isregression=True,
ismuticlass=False,
sdo=False,
sdo_frac=0.2):
"""
This is an encoder-decoder network architecture for image reconstruction and image segmentation
based on pretrained NASNetLarge model as encoder. The deocder need to be trained
:param img_sz: size of the input image [n x n x 3]
:param noutchannel: The number of output channels of the network, only valid if ismuticlass==True else ignored
:param isregression: If ==True then model output is single channel without any activation applied to the last layer, ismuticlass and noutchannel is ignored
If ==False and ismuticlass==False, then output is single channel and sigmoid activation applied to the last layer
If ==False and ismuticlass==True, then output is multichannel determined by noutchannel and softmax activation is appled to the last layer
:param ismuticlass: If ==False then output is single channel and sigmoid activation applied to the last layer
If ==True then output is multichannel determined by noutchannel and softmax activation is appled to the last layer
:return: Encoder Decoder Model
"""
model = NASNetMobile(include_top=False,
weights=weights,
input_tensor=None,
input_shape=(H, W, 3),
pooling=None)
# ----- Decoder Network ------ #
nFeat_d = 256
convT_0_0_d = UpSampling2D_BN_Act(
kSize=2,
crop=0,
outPad=0,
concateInp=add([
model.get_layer('activation_129').output,
model.get_layer('activation_130').output
]),
inp=model.get_layer('activation_188').output)
conv_0_0_d = conv2D_BN_Act(nFeature=nFeat_d,
kSize=3,
kStride=1,
inp=convT_0_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
conv_0_1_d = conv2D_BN_Act(nFeature=nFeat_d,
kSize=3,
kStride=1,
inp=conv_0_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
convT_1_0_d = UpSampling2D_BN_Act(
kSize=2,
crop=0,
outPad=0,
concateInp=add([
model.get_layer('activation_70').output,
model.get_layer('activation_71').output
]),
inp=conv_0_1_d)
conv_1_0_d = conv2D_BN_Act(nFeature=nFeat_d,
kSize=3,
kStride=1,
inp=convT_1_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
conv_1_1_d = conv2D_BN_Act(nFeature=nFeat_d,
kSize=3,
kStride=1,
inp=conv_1_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
convT_2_0_d = UpSampling2D_BN_Act(
kSize=2,
crop=0,
outPad=0,
concateInp=add([
model.get_layer('activation_13').output,
model.get_layer('activation_15').output,
model.get_layer('activation_17').output,
model.get_layer('activation_19').output
]),
inp=conv_1_1_d)
conv_2_0_d = conv2D_BN_Act(nFeature=nFeat_d,
kSize=3,
kStride=1,
inp=convT_2_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
conv_2_1_d = conv2D_BN_Act(nFeature=nFeat_d,
kSize=3,
kStride=1,
inp=conv_2_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
convT_3_0_d = UpSampling2D_BN_Act(
kSize=2,
crop=((0, 1), (0, 1)),
outPad=0,
concateInp=add([
model.get_layer('activation_1').output,
model.get_layer('activation_4').output,
model.get_layer('activation_6').output,
model.get_layer('activation_8').output
]),
inp=conv_2_1_d)
conv_3_0_d = conv2D_BN_Act(nFeature=nFeat_d,
kSize=3,
kStride=1,
inp=convT_3_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
conv_3_1_d = conv2D_BN_Act(nFeature=nFeat_d,
kSize=3,
kStride=1,
inp=conv_3_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
convT_4_0_d = UpSampling2D_BN_Act(kSize=2,
crop=0,
outPad=1,
concateInp=model.input,
inp=conv_3_1_d)
conv_4_0_d = conv2D_BN_Act(nFeature=nFeat_d / 2,
kSize=3,
kStride=1,
inp=convT_4_0_d,
sdo=sdo,
sdo_frac=sdo_frac)
if (isregression):
conv_4_1_d = Conv2D(filters=1,
kernel_size=3,
strides=1,
padding='same')(conv_4_0_d)
else:
if (ismuticlass):
conv_4_1_d = Conv2D(filters=noutchannel,
kernel_size=3,
strides=1,
padding='same')(conv_4_0_d)
conv_4_1_d = Activation('softmax')(conv_4_1_d)
else:
conv_4_1_d = Conv2D(filters=1,
kernel_size=3,
strides=1,
padding='same')(conv_4_0_d)
conv_4_1_d = Activation('sigmoid')(conv_4_1_d)
model_1 = Model(inputs=[model.input], outputs=[conv_4_1_d])
if ninchannel != 3:
model_1 = append_inputchannels(model_1, H, W, ninchannel)
return model_1
def Build_model(self):
############# Feature Extraction Network ##################
if self.Backbone_model == "VGG16":
Backbone = VGG16(input_shape=(352,352,3), include_top=False, weights='imagenet')
Stages=[Backbone.get_layer("block5_conv3").output, Backbone.get_layer("block4_conv3").output,
Backbone.get_layer("block3_conv3").output]
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
elif self.Backbone_model == "ResNet50":
Backbone = ResNet50(input_shape=(352,352,3), include_top=False, weights='imagenet')
Stages=[Backbone.output, Backbone.get_layer("conv4_block6_out").output,
Backbone.get_layer("conv3_block4_out").output, Backbone.get_layer("conv2_block3_out").output]
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
elif self.Backbone_model == "NASNetMobile":
Backbone = NASNetMobile(input_shape=(352,352,3), include_top=False, weights=None)
if os.path.exists('./NASNet-mobile-no-top.h5'):
pass
else:
urlretrieve('https://github.com/titu1994/Keras-NASNet/releases/download/v1.2/NASNet-mobile-no-top.h5',
'./NASNet-mobile-no-top.h5')
Backbone.load_weights('./NASNet-mobile-no-top.h5')
Stages=[Backbone.output, Backbone.get_layer("activation_131").output,
Backbone.get_layer("activation_72").output, Backbone.get_layer("activation_13").output]
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
elif self.Backbone_model == "NASNetLarge":
Backbone = NASNetLarge(input_shape=(352,352,3), include_top=False, weights=None)
if os.path.exists('./NASNet-large-no-top.h5'):
pass
else:
urlretrieve('https://github.com/titu1994/Keras-NASNet/releases/download/v1.2/NASNet-large-no-top.h5',
'./NASNet-large-no-top.h5')
Backbone.load_weights('./NASNet-large-no-top.h5')
Stages=[Backbone.output, Backbone.get_layer("activation_319").output,
Backbone.get_layer("activation_260").output, Backbone.get_layer("activation_201").output]
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
else:
raise ValueError("The name of the Backbone_model must be one of the following options: VGG16, ResNet50, NASNetMobile, NASNetLarge")
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
ExtractedFeatures = []
Number_of_Filters = [192,128,96,64]
for Stage,Num in zip(Stages,Number_of_Filters):
MAG_output=MAG_Module(Stage, Num)
extr=Conv2D(Num, (1, 1), padding='same')(MAG_output)
ExtractedFeatures.append(extr) #Extracted features from the Feature Extraction Network
############### Feature Integration Network ##################
z = BilinearUpsampling(output_size=(ExtractedFeatures[0]._keras_shape[1]*2,
ExtractedFeatures[0]._keras_shape[2]*2))(ExtractedFeatures[0])
for i in range(len(ExtractedFeatures)-1):
z = AMI_Module(z, ExtractedFeatures[i+1], ExtractedFeatures[i+1]._keras_shape[-1])
z = BilinearUpsampling(output_size=(z._keras_shape[1]*2,z._keras_shape[2]*2))(z)
z = Conv2D(64, (3, 3), padding='same', strides=(1,1))(z)
z = BatchNormalization(axis=-1)(z)
z = Activation('relu')(z)
z = BilinearUpsampling(output_size=(352,352))(z)
z = Conv2D(2, (1, 1), padding='same')(z)
z = Activation('softmax')(z)
self.model = Model(Backbone.input,z)
self.model.compile(optimizer=SGD(lr=self.learning_rate, momentum=0.9), loss=Sharpenning_Loss(self.Lambda), metrics=["accuracy"])
if self.show_ModelSummary == True:
self.model.summary()