def _build_graph(self):
assert self._model_type in self.Types, f'model type unrecognised: {self._model_type} known types: {self.Types}'
with tf.variable_scope('perceptual_model_' + self._model_type):
if self._model_type == 'vgg16':
self._model = apps.VGG16(include_top=False,
weights='imagenet',
input_shape=self._img_shape)
elif self._model_type == 'vgg19':
self._model = apps.VGG19(include_top=False,
weights='imagenet',
input_shape=self._img_shape)
outputs = [self._model.get_layer(l).output for l in self._layers]
self._feature_model = Model(self._model.input, outputs)
self._freeze_all_vars_in_scope()
def get_backbone(backbone,
input_tensor,
use_imagenet=False,
return_dict=True,
**kwargs):
"""Retrieve backbones - helper function for the construction of feature pyramid networks
backbone: Name of the backbone to be retrieved. Options include featurenets, resnets
densenets, mobilenets, and nasnets
input_tensor: The tensor to be used as the input for the backbone. Should have channel
dimension of size 3
use_imagenet: Defaults to False. Whether to load pre-trained weights for the backbone
return_dict: Defaults to True. Whether to return a dictionary of backbone layers,
e.g. {'C1': C1, 'C2': C2, 'C3': C3, 'C4': C4, 'C5': C5}. If false, the whole model
is returned instead
kwargs: Keyword dictionary for backbone constructions.
Relevant keys include 'include_top', 'weights' (should be set to None),
'input_shape', and 'pooling'
"""
_backbone = str(backbone).lower()
featurenet_backbones = ['featurenet', 'featurenet3d', 'featurenet_3d']
vgg_backbones = ['vgg16', 'vgg19']
densenet_backbones = ['densenet121', 'densenet169', 'densenet201']
mobilenet_backbones = ['mobilenet', 'mobilenetv2', 'mobilenet_v2']
resnet_backbones = ['resnet50']
nasnet_backbones = ['nasnet_large', 'nasnet_mobile']
# TODO: Check and make sure **kwargs is in the right format.
# 'weights' flag should be None, and 'input_shape' must have size 3 on the channel axis
if use_imagenet:
kwargs_with_weights = copy.copy(kwargs)
kwargs_with_weights['weights'] = 'imagenet'
if _backbone in featurenet_backbones:
if use_imagenet:
raise ValueError('A featurenet backbone that is pre-trained on '
'imagenet does not exist')
if '3d' in _backbone:
model, output_dict = featurenet_3D_backbone(
input_tensor=input_tensor, **kwargs)
else:
model, output_dict = featurenet_backbone(input_tensor=input_tensor,
**kwargs)
if return_dict:
return output_dict
else:
return model
if _backbone in vgg_backbones:
if _backbone == 'vgg16':
model = applications.VGG16(input_tensor=input_tensor, **kwargs)
else:
model = applications.VGG19(input_tensor=input_tensor, **kwargs)
# Set the weights of the model if requested
if use_imagenet:
if _backbone == 'vgg16':
model_with_weights = applications.VGG16(**kwargs_with_weights)
else:
model_with_weights = applications.VGG19(**kwargs_with_weights)
model_with_weights.save_weights('model_weights.h5')
model.load_weights('model_weights.h5', by_name=True)
layer_names = [
'block1_pool', 'block2_pool', 'block3_pool', 'block4_pool',
'block5_pool'
]
layer_outputs = [
model.get_layer(name=layer_name).output
for layer_name in layer_names
]
output_dict = {}
for i, j in enumerate(layer_names):
output_dict['C' + str(i + 1)] = layer_outputs[i]
if return_dict:
return output_dict
else:
return model
elif _backbone in densenet_backbones:
if _backbone == 'densenet121':
model = applications.DenseNet121(input_tensor=input_tensor,
**kwargs)
blocks = [6, 12, 24, 16]
elif _backbone == 'densenet169':
model = applications.DenseNet169(input_tensor=input_tensor,
**kwargs)
blocks = [6, 12, 32, 32]
elif _backbone == 'densenet201':
model = applications.DenseNet201(input_tensor=input_tensor,
**kwargs)
blocks = [6, 12, 48, 32]
# Set the weights of the model if requested
if use_imagenet:
if _backbone == 'densenet121':
model_with_weights = applications.DenseNet121(
**kwargs_with_weights)
elif _backbone == 'densenet169':
model_with_weights = applications.DenseNet169(
**kwargs_with_weights)
elif _backbone == 'densenet201':
model_with_weights = applications.DenseNet201(
**kwargs_with_weights)
model_with_weights.save_weights('model_weights.h5')
model.load_weights('model_weights.h5', by_name=True)
layer_names = ['conv1/relu'] + [
'conv{}_block{}_concat'.format(idx + 2, block_num)
for idx, block_num in enumerate(blocks)
]
layer_outputs = [
model.get_layer(name=layer_name).output
for layer_name in layer_names
]
output_dict = {}
for i, j in enumerate(layer_names):
output_dict['C%' + str(i + 1)] = layer_outputs[i]
if return_dict:
return output_dict
else:
return model
elif _backbone in resnet_backbones:
model = applications.ResNet50(input_tensor=input_tensor, **kwargs)
# Set the weights of the model if requested
if use_imagenet:
model_with_weights = applications.ResNet50(**kwargs_with_weights)
model_with_weights.save_weights('model_weights.h5')
model.load_weights('model_weights.h5', by_name=True)
layer_names = [
'bn_conv1', 'res2c_branch2c', 'res3d_branch2c', 'res4f_branch2c',
'res5c_branch2c'
]
layer_outputs = [
model.get_layer(name=layer_name).output
for layer_name in layer_names
]
output_dict = {}
for i, j in enumerate(layer_names):
output_dict['C' + str(i + 1)] = layer_outputs[i]
if return_dict:
return output_dict
else:
return model
elif _backbone in mobilenet_backbones:
alpha = kwargs.get('alpha', 1.0)
if _backbone.endswith('v2'):
model = applications.MobileNetV2(alpha=alpha,
input_tensor=input_tensor,
**kwargs)
block_ids = (2, 5, 12)
layer_names = ['expanded_conv_project_BN'] + \
['block_%s_add' % i for i in block_ids] + \
['block_16_project_BN']
else:
model = applications.MobileNet(alpha=alpha,
input_tensor=input_tensor,
**kwargs)
block_ids = (1, 3, 5, 11, 13)
layer_names = ['conv_pw_%s_relu' % i for i in block_ids]
# Set the weights of the model if requested
if use_imagenet:
if _backbone.endswith('v2'):
model_with_weights = applications.MobileNetV2(
alpha=alpha, **kwargs_with_weights)
else:
model_with_weights = applications.MobileNet(
alpha=alpha, **kwargs_with_weights)
model_with_weights.save_weights('model_weights.h5')
model.load_weights('model_weights.h5', by_name=True)
layer_outputs = [
model.get_layer(name=layer_name).output
for layer_name in layer_names
]
output_dict = {}
for i, j in enumerate(layer_names):
output_dict['C' + str(i + 1)] = layer_outputs[i]
if return_dict:
return output_dict
else:
return model
elif _backbone in nasnet_backbones:
if _backbone.endswith('large'):
model = applications.NASNetLarge(input_tensor=input_tensor,
**kwargs)
block_ids = [5, 12, 18]
else:
model = applications.NASNetMobile(input_tensor=input_tensor,
**kwargs)
block_ids = [3, 8, 12]
# Set the weights of the model if requested
if use_imagenet:
if _backbone.endswith('large'):
model_with_weights = applications.NASNetLarge(
**kwargs_with_weights)
else:
model_with_weights = applications.NASNetMobile(
**kwargs_with_weights)
model_with_weights.save_weights('model_weights.h5')
model.load_weights('model_weights.h5', by_name=True)
layer_names = ['stem_bn1', 'reduction_concat_stem_1']
layer_names += ['normal_concat_%s' % i for i in block_ids]
layer_outputs = [
model.get_layer(name=layer_name).output
for layer_name in layer_names
]
output_dict = {}
for i, j in enumerate(layer_names):
output_dict['C' + str(i + 1)] = layer_outputs[i]
if return_dict:
return output_dict
else:
return model
else:
backbones = list(featurenet_backbones + densenet_backbones +
resnet_backbones + vgg_backbones + nasnet_backbones)
raise ValueError('Invalid value for `backbone`. Must be one of: %s' %
', '.join(backbones))
config = tf.ConfigProto(device_count={'GPU': 1, 'CPU': 56})
sess = tf.Session(config=config)
k.set_session(sess)
img_width, img_height = 192, 192
train_data_dir = "C:\\Users\\arsee\\Desktop\\NN2\\training_set"
validation_data_dir = "C:\\Users\\arsee\\Desktop\\NN2\\test_set"
nb_train_samples = 2147
nb_validation_samples = 269
batch_size = 16
epochs = 8
model = applications.VGG19(weights="imagenet",
include_top=False,
input_shape=(img_width, img_height, 3))
"""
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) (None, 256, 256, 3) 0
_________________________________________________________________
block1_conv1 (Conv2D) (None, 256, 256, 64) 1792
_________________________________________________________________
block1_conv2 (Conv2D) (None, 256, 256, 64) 36928
_________________________________________________________________
block1_pool (MaxPooling2D) (None, 128, 128, 64) 0
_________________________________________________________________
block2_conv1 (Conv2D) (None, 128, 128, 128) 73856
_________________________________________________________________
block2_conv2 (Conv2D) (None, 128, 128, 128) 147584
def get_pyramid_layer_outputs(backbone, inputs, **kwargs):
_backbone = str(backbone).lower()
vgg_backbones = {'vgg16', 'vgg19'}
densenet_backbones = {'densenet121', 'densenet169', 'densenet201'}
mobilenet_backbones = {'mobilenet', 'mobilenetv2', 'mobilenet_v2'}
resnet_backbones = {'resnet50'}
nasnet_backbones = {'nasnet_large', 'nasnet_mobile'}
## 3D ADDING
resnet_3D_backbones = {'resnet50_3d', 'resnet50_3D_gne'}
if _backbone in vgg_backbones:
layer_names = ['block3_pool', 'block4_pool', 'block5_pool']
if _backbone == 'vgg16':
model = applications.VGG16(**kwargs)
else:
model = applications.VGG19(**kwargs)
return [model.get_layer(n).output for n in layer_names]
elif _backbone in densenet_backbones:
if _backbone == 'densenet121':
model = applications.DenseNet121(**kwargs)
blocks = [6, 12, 24, 16]
elif _backbone == 'densenet169':
model = applications.DenseNet169(**kwargs)
blocks = [6, 12, 32, 32]
elif _backbone == 'densenet201':
model = applications.DenseNet201(**kwargs)
blocks = [6, 12, 48, 32]
layer_outputs = []
for idx, block_num in enumerate(blocks):
name = 'conv{}_block{}_concat'.format(idx + 2, block_num)
layer_outputs.append(model.get_layer(name=name).output)
# create the densenet backbone
model = Model(inputs=inputs,
outputs=layer_outputs[1:],
name=model.name)
return model.outputs
elif _backbone in resnet_backbones:
model = applications.ResNet50(**kwargs)
for i in kwargs:
print(i, kwargs[i])
layer_names = ['res3d_branch2c', 'res4f_branch2c', 'res5c_branch2c']
layer_outputs = [model.get_layer(name).output for name in layer_names]
model = Model(inputs=inputs, outputs=layer_outputs, name=model.name)
return model.outputs
## 3D ADDING
elif _backbone in resnet_3D_backbones:
for i in kwargs:
print(i, kwargs[i])
model = application_3D.ResNet50_3D(**kwargs)
layer_names = ['res3d_branch2c', 'res4f_branch2c', 'res5c_branch2c']
layer_outputs = [model.get_layer(name).output for name in layer_names]
model = Model(inputs=inputs, outputs=layer_outputs, name=model.name)
return model.outputs
elif _backbone in mobilenet_backbones:
alpha = kwargs.get('alpha', 1.0)
if _backbone.endswith('v2'):
model = applications.MobileNetV2(alpha=alpha, **kwargs)
block_ids = (12, 15, 16)
layer_names = ['block_%s_depthwise_relu' % i for i in block_ids]
else:
model = applications.MobileNet(alpha=alpha, **kwargs)
block_ids = (5, 11, 13)
layer_names = ['conv_pw_%s_relu' % i for i in block_ids]
layer_outputs = [model.get_layer(name).output for name in layer_names]
model = Model(inputs=inputs, outputs=layer_outputs, name=model.name)
return model.outputs
elif _backbone in nasnet_backbones:
if _backbone.endswith('large'):
model = applications.NASNetLarge(**kwargs)
block_ids = [5, 12, 18]
else:
model = applications.NASNetMobile(**kwargs)
block_ids = [3, 8, 12]
layer_names = ['normal_conv_1_%s' % i for i in block_ids]
layer_outputs = [model.get_layer(name).output for name in layer_names]
model = Model(inputs=inputs, outputs=layer_outputs, name=model.name)
return model.outputs
else:
backbones = list(densenet_backbones + resnet_backbones +
vgg_backbones + resnet_3D_backbones)
raise ValueError('Invalid value for `backbone`. Must be one of: %s' %
', '.join(backbones))
def get_vgg_layers(layer_names):
vgg = applications.VGG19(include_top=False, weights='imagenet')
vgg.trainable = False
outputs = [vgg.get_layer(name).output for name in layer_names]
model = models.Model([vgg.input], outputs)
return model