def collect_models():
models = dict()
models["MobileNetV2"] = tfapp.MobileNetV2(input_shape=IMG_SHAPE,
include_top=False,
weights='imagenet')
models["NASNetMobile"] = tfapp.NASNetMobile(input_shape=(130, 386, 3),
include_top=False,
weights='imagenet')
models["DenseNet121"] = tfapp.DenseNet121(input_shape=IMG_SHAPE,
include_top=False,
weights='imagenet')
models["VGG16"] = tfapp.VGG16(input_shape=IMG_SHAPE,
include_top=False,
weights='imagenet')
models["Xception"] = tfapp.Xception(input_shape=(134, 390, 3),
include_top=False,
weights='imagenet')
models["ResNet50V2"] = tfapp.ResNet50V2(input_shape=IMG_SHAPE,
include_top=False,
weights='imagenet')
models["NASNetLarge"] = tfapp.NASNetLarge(input_shape=(130, 386, 3),
include_top=False,
weights='imagenet')
# omit non 2^n shape
# models["InceptionV3"] = tfapp.InceptionV3(input_shape=IMG_SHAPE, include_top=False, weights='imagenet')
# models["InceptionResNetV2"] = \
# tfapp.InceptionResNetV2(input_shape=IMG_SHAPE, include_top=False, weights='imagenet')
return models
def make_new_model():
resnet50 = tfapp.ResNet50V2(input_shape=IMG_SHAPE,
include_top=False,
weights='imagenet')
midout = resnet50.get_layer('conv3_block4_out').output
output = convolution(midout, 128, 3, 1, 'custom_output')
model = tf.keras.Model(resnet50.input, outputs=output)
model.summary()
def build(self):
"""
Build the model
input shape: [batch_size, w, h, channels]
"""
inputs = layers.Input(self.config.model.input_shape)
x = inputs
batch_norm = self.config.model.batch_norm
base_model = applications.ResNet50V2(
input_shape=self.config.model.input_shape, include_top=False)
# Use the activations of these layers
layer_names = [
'conv1_conv',
'conv2_block2_out', # 48 x 48 x 256
'conv3_block3_out', # 24 x 24 x 512
'conv4_block5_out', # 12 x 12 x 1024
'conv5_block3_out', # 6 x 6 x 2048
]
outputs = [base_model.get_layer(name).output for name in layer_names]
output_channels = [
base_model.get_layer(name).output_shape[-1] for name in layer_names
]
# Create the feature extraction model
down_stack = models.Model(inputs=base_model.input,
outputs=outputs,
name="ResNet50V2")
down_stack.trainable = False
skips = down_stack(x)
x = skips[-1]
for i, (skip, k) in enumerate(
zip(reversed(skips[:-1]), reversed(output_channels[:-1]))):
upsample = _upconvolution(k,
batch_norm=batch_norm,
name=f"up_conv2d_{i}")
x = upsample(x)
concat = layers.Concatenate(axis=3)
x = concat([skip, x])
block = _upsample_block(k,
batch_norm=batch_norm,
name=f"upsampler_block_{i}",
padding="same")
x = block(x)
output_upsample = _upconvolution(output_channels[0],
batch_norm=batch_norm,
name=f"up_conv2d_out")
x = output_upsample(x)
output_conv = layers.Conv2D(self.config.model.num_classes, 1)
x = output_conv(x)
flatten = layers.Reshape((-1, self.config.model.num_classes))
outputs = flatten(x)
self.model = models.Model(inputs=inputs, outputs=outputs)
def rn50(img_size=(224,224), num_class=2, weights="imagenet", dtype=tf.float32):
input_layer = layers.Input(shape=(img_size[0],img_size[1],3), dtype=dtype)
base = models.ResNet50V2(input_tensor=input_layer, include_top=False, weights=weights)
base.trainable = True
x = base.output
x = layers.GlobalAveragePooling2D()(x)
x = layers.Dense(num_class)(x)
preds = layers.Activation("softmax", dtype=tf.float32)(x)
model = tf.keras.models.Model(inputs=input_layer, outputs=preds)
return model
def ResNet50V2(num_classes: int, image_size=(256, 256)):
base_model = applications.ResNet50V2(
weights=None, include_top=False, input_shape=(*image_size, 3))
x = base_model.output
x = layers.GlobalAveragePooling2D()(x)
x = layers.Dense(512, activation='relu')(x)
x = layers.Dropout(0.5)(x)
predictions = layers.Dense(num_classes, activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=predictions)
return model
return op.get(type)
all_combinations = [(bs, it, op, lr) for bs in BATCH_SIZE
for it in INTERPOLATION for op in OPTIMIZER_TYPE
for lr in LEARNING_RATE]
for bs, it, op, lr in all_combinations:
train_data, valid_data = load_preprocess(DIRECTORY, bs, IMAGE_SIZE, it)
print(
"Parameter Tuning: Batch_size = {}, Interpolation = {}, Optimizer = {}, Learning_rate = {}"
.format(bs, it, op, lr))
base_model = applications.ResNet50V2(include_top=False,
weights="imagenet",
input_shape=(224, 224, 3))
base_model.summary()
train_feat, train_labels = extract_features(base_model, train_data, 1442,
bs, 2048)
valid_feat, valid_labels = extract_features(base_model, valid_data, 618,
bs, 2048)
inputs = keras.Input(shape=(7, 7, 2048))
x = layers.Flatten()(inputs)
x = layers.Dense(2048, activation='relu')(x)
#x = layers.Dropout(0.3)(x)
x = layers.Dense(512, activation='relu')(x)
#x = layers.Dropout(0.3)(x)
x = layers.Dense(128, activation='relu')(x)
def make_model(args, nclass, input_shape):
# Weight decay
if args.weight_decay is not None and args.weight_decay > 0:
regularizer = keras.regularizers.L2(args.weight_decay / 2)
else:
regularizer = None
if args.optimizer == 'lamb':
regularizer = None
logging.info(
'adding weight decay via the LAMB optimizer instead of Keras regularization'
)
# Inputs
input = keras.Input(input_shape, name='image')
input2 = keras.Input(input_shape, name='image2')
if args.backbone == 'resnet50v2':
backbone = applications.ResNet50V2(weights=None,
include_top=False,
input_shape=input_shape,
pooling='avg')
if (input_shape[0] or 224) < 224:
logging.warning('using standard resnet on small dataset')
elif args.backbone == 'small-resnet50v2':
backbone = small_resnet_v2.SmallResNet50V2(include_top=False,
input_shape=input_shape,
pooling='avg')
if (input_shape[0] or 224) >= 224:
logging.warning('using small resnet on large dataset')
elif args.backbone == 'resnet50':
backbone = applications.ResNet50(weights=None,
include_top=False,
input_shape=input_shape,
pooling='avg')
elif args.backbone == 'affine':
backbone = keras.Sequential(
[layers.GlobalAveragePooling2D(),
layers.Dense(1)])
else:
raise Exception(f'unknown model {args.backbone}')
# Add weight decay to backbone
backbone = add_regularization_with_reset(backbone, regularizer)
# Standardize input
stand_img = custom_layers.StandardizeImage()
# Features
raw_feats, raw_feats2 = backbone(stand_img(input)), backbone(
stand_img(input2))
# Normalize features?
feats, feats2 = optional_normalize(args.feat_norm, raw_feats, raw_feats2)
# Name the features
feats = custom_layers.Identity(name='feats')(feats)
feats2 = custom_layers.Identity(name='feats2')(feats2)
# Measure the norms of the features
if args.feat_norm is not None:
feats = custom_layers.MeasureNorm(name='feat_norm')(feats)
# Projection
if args.proj_dim is None or args.proj_dim <= 0:
projection = custom_layers.Identity(name='projection')
else:
projection = tf.keras.Sequential([
layers.Dense(2048,
kernel_regularizer=regularizer,
bias_regularizer=regularizer),
layers.BatchNormalization(),
layers.ReLU(),
layers.Dense(2048,
kernel_regularizer=regularizer,
bias_regularizer=regularizer),
layers.BatchNormalization(),
layers.ReLU(),
layers.Dense(
args.proj_dim, kernel_regularizer=regularizer, use_bias=False)
],
name='projection')
if args.proj_norm == 'sn':
projection = custom_layers.SpectralNormalization(projection,
name='sn_projection')
# Projected features
proj_feats, proj_feats2 = projection(feats), projection(feats2)
# Normalize projected features?
proj_feats, proj_feats2 = optional_normalize(args.proj_norm, proj_feats,
proj_feats2)
# Name the projected features
proj_feats = custom_layers.Identity(name='proj_feats')(proj_feats)
proj_feats2 = custom_layers.Identity(name='proj_feats2')(proj_feats2)
# Measure the norms of the projected features
if args.proj_dim is not None and args.proj_dim > 0:
proj_feats = custom_layers.MeasureNorm(name='proj_norm')(proj_feats)
# Feature views
proj_views = custom_layers.FeatViews(name='contrast', dtype=tf.float32)(
(proj_feats, proj_feats2))
# Label logits
if args.stop_gradient:
feats = tf.stop_gradient(feats)
prediction = layers.Dense(
nclass,
name='label',
kernel_regularizer=regularizer if args.optimizer != 'lamb' else None,
bias_regularizer=regularizer if args.optimizer != 'lamb' else None,
dtype=tf.float32)(feats)
# Model
inputs = [input, input2]
outputs = [prediction, proj_views]
model = keras.Model(inputs, outputs)
return model
def encode(self, input_image):
"""
:param input_image: (batch, height, width, channel)
"""
input_shape = input_image.get_shape()
height = input_shape[1]
net_name = self.net_name
weights = "imagenet" if self.pretrained_weight else None
jsonfile = op.join(opts.PROJECT_ROOT, "model", "build_model",
"scaled_layers.json")
output_layers = self.read_output_layers(jsonfile)
out_layer_names = output_layers[net_name]
if net_name == "MobileNetV2":
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input
pproc_img = layers.Lambda(lambda x: preprocess_input(x),
name="preprocess_mobilenet")(input_image)
ptmodel = tfapp.MobileNetV2(input_shape=input_shape,
include_top=False,
weights=weights)
elif net_name == "NASNetMobile":
from tensorflow.keras.applications.nasnet import preprocess_input
assert height == 128
def preprocess_layer(x):
x = preprocess_input(x)
x = tf.image.resize(x,
size=NASNET_SHAPE[:2],
method="bilinear")
return x
pproc_img = layers.Lambda(lambda x: preprocess_layer(x),
name="preprocess_nasnet")(input_image)
ptmodel = tfapp.NASNetMobile(input_shape=NASNET_SHAPE,
include_top=False,
weights=weights)
elif net_name == "DenseNet121":
from tensorflow.keras.applications.densenet import preprocess_input
pproc_img = layers.Lambda(lambda x: preprocess_input(x),
name="preprocess_densenet")(input_image)
ptmodel = tfapp.DenseNet121(input_shape=input_shape,
include_top=False,
weights=weights)
elif net_name == "VGG16":
from tensorflow.keras.applications.vgg16 import preprocess_input
pproc_img = layers.Lambda(lambda x: preprocess_input(x),
name="preprocess_vgg16")(input_image)
ptmodel = tfapp.VGG16(input_shape=input_shape,
include_top=False,
weights=weights)
elif net_name == "Xception":
from tensorflow.keras.applications.xception import preprocess_input
assert height == 128
def preprocess_layer(x):
x = preprocess_input(x)
x = tf.image.resize(x,
size=XCEPTION_SHAPE[:2],
method="bilinear")
return x
pproc_img = layers.Lambda(lambda x: preprocess_layer(x),
name="preprocess_xception")(input_image)
ptmodel = tfapp.Xception(input_shape=XCEPTION_SHAPE,
include_top=False,
weights=weights)
elif net_name == "ResNet50V2":
from tensorflow.keras.applications.resnet import preprocess_input
pproc_img = layers.Lambda(lambda x: preprocess_input(x),
name="preprocess_resnet")(input_image)
ptmodel = tfapp.ResNet50V2(input_shape=input_shape,
include_top=False,
weights=weights)
elif net_name == "NASNetLarge":
from tensorflow.keras.applications.nasnet import preprocess_input
assert height == 128
def preprocess_layer(x):
x = preprocess_input(x)
x = tf.image.resize(x,
size=NASNET_SHAPE[:2],
method="bilinear")
return x
pproc_img = layers.Lambda(lambda x: preprocess_layer(x),
name="preprocess_nasnet")(input_image)
ptmodel = tfapp.NASNetLarge(input_shape=NASNET_SHAPE,
include_top=False,
weights=weights)
else:
raise WrongInputException("Wrong pretrained model name: " +
net_name)
# collect multi scale convolutional features
layer_outs = []
for layer_name in out_layer_names:
layer = ptmodel.get_layer(name=layer_name[1], index=layer_name[0])
# print("extract feature layers:", layer.name, layer.get_input_shape_at(0), layer.get_output_shape_at(0))
layer_outs.append(layer.output)
# create model with multi scale features
multi_scale_model = tf.keras.Model(ptmodel.input,
layer_outs,
name=net_name + "_base")
features_ms = multi_scale_model(pproc_img)
return features_ms