def compute_mean_and_std(model_name, X, input_shape):
if model_name == 'Xception':
model = applications.Xception(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'VGG16':
model = applications.VGG16(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'VGG19':
model = applications.VGG19(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'ResNet50':
model = applications.ResNet50(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'InceptionResNetV2':
model = applications.InceptionResNetV2(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'InceptionV3':
model = applications.InceptionV3(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'MobileNet':
model = applications.MobileNet(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'DenseNet121':
model = applications.DenseNet121(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'DenseNet169':
model = applications.DenseNet169(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'DenseNet201':
model = applications.DenseNet201(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'NASNetMobile':
model = applications.NASNetMobile(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif model_name == 'NASNetLarge':
model = applications.NASNetLarge(weights='imagenet',
include_top=False,
input_shape=input_shape)
else:
assert (False), "Specified base model is not available !"
features = model.predict(X)[:, 0, 0, :]
return features.mean(axis=0), features.std(axis=0)
# ### Initialization
model = Sequential()
model.add(InputLayer(
input_shape=INPUT_IMAGE_SIZE)) # possibly needed due to a bug in Keras
if pretrained == 'VGG16':
pt_model = applications.VGG16(weights='imagenet',
include_top=False,
input_shape=INPUT_IMAGE_SIZE)
pretrained_first_trainable_layer = 15
elif pretrained == 'MobileNet':
pt_model = applications.MobileNet(weights='imagenet',
include_top=False,
input_shape=INPUT_IMAGE_SIZE)
pretrained_first_trainable_layer = 75
else:
assert 0, "Unknown model: " + pretrained
pt_name = pt_model.name
print('Using {} pre-trained model'.format(pt_name))
for layer in pt_model.layers:
model.add(layer)
for layer in model.layers:
layer.trainable = False
print(model.summary())
def build_autoencoder(base_model_name, input_shape, imagenet_mean,
imagenet_std, hidden_layer_size, n_classes,
weight_decay):
if base_model_name == 'Xception':
base_model = applications.Xception(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'VGG16':
base_model = applications.VGG16(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'VGG19':
base_model = applications.VGG19(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'ResNet50':
base_model = applications.ResNet50(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'InceptionResNetV2':
base_model = applications.InceptionResNetV2(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'InceptionV3':
base_model = applications.InceptionV3(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'MobileNet':
base_model = applications.MobileNet(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'DenseNet121':
base_model = applications.DenseNet121(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'DenseNet169':
base_model = applications.DenseNet169(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'DenseNet201':
base_model = applications.DenseNet201(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'NASNetMobile':
base_model = applications.NASNetMobile(weights='imagenet',
include_top=False,
input_shape=input_shape)
elif base_model_name == 'NASNetLarge':
base_model = applications.NASNetLarge(weights='imagenet',
include_top=False,
input_shape=input_shape)
else:
assert (False), "Specified base model is not available !"
n_features = base_model.output.shape[-1]
x = base_model.output
x = tf.keras.layers.Lambda(lambda x: (x - imagenet_mean) / imagenet_std)(
x) # normalization
x = tf.keras.layers.Activation(activation='sigmoid',
name='encoder')(x) # sigmoid
x = tf.keras.layers.Dense(units=hidden_layer_size,
activation=None)(x) # encoding
x = tf.keras.layers.Activation(activation='relu')(x) # relu
x = tf.keras.layers.Dense(units=n_features,
activation=None,
name='decoder')(x) # decoding
x = tf.keras.layers.Dense(units=n_classes, activation='sigmoid')(
x) # x = tf.keras.layers.Activation(activation='sigmoid')(x) # sigmoid
model = tf.keras.Model(inputs=base_model.input, outputs=x[:, 0, 0, :])
for layer in model.layers:
if isinstance(layer, tf.keras.layers.Conv2D) or isinstance(
layer, tf.keras.layers.Dense):
layer.add_loss(
tf.keras.regularizers.l2(weight_decay)(layer.kernel))
if hasattr(layer, 'bias_regularizer') and layer.use_bias:
layer.add_loss(tf.keras.regularizers.l2(weight_decay)(layer.bias))
return model
def train(base_dir):
#%% Init model
encoder = keras_applications.MobileNet(input_shape=(224, 224, 3),
include_top=False,
pooling="avg")
support_layer = GramMatrix(kernel={
"name": "MixedNorms",
"init": {
"norms": [
lambda x: 1 - tf.nn.l2_normalize(x[0]) * tf.nn.l2_normalize(x[
1]),
lambda x: tf.math.abs(x[0] - x[1]),
lambda x: tf.nn.softmax(tf.math.abs(x[0] - x[1])),
lambda x: tf.square(x[0] - x[1]),
],
"use_bias":
True,
},
}, )
model = Sequential([encoder, support_layer])
#%% Init training
callbacks = [
TensorBoard(base_dir, write_images=True, histogram_freq=1),
ModelCheckpoint(str(base_dir / "best_loss.h5"), save_best_only=True),
ReduceLROnPlateau(),
]
#%% Init data
@tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3],
dtype=tf.uint8), ))
def preprocessing(input_tensor):
output_tensor = tf.cast(input_tensor, dtype=tf.float32)
output_tensor = tf.image.resize_with_pad(output_tensor,
target_height=224,
target_width=224)
output_tensor = keras_applications.mobilenet.preprocess_input(
output_tensor, data_format="channels_last")
return output_tensor
@tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3],
dtype=tf.float32), ))
def data_augmentation(input_tensor):
output_tensor = tf.image.random_flip_left_right(input_tensor)
output_tensor = tf.image.random_flip_up_down(output_tensor)
output_tensor = tf.image.random_brightness(output_tensor,
max_delta=0.25)
return preprocessing(output_tensor)
all_annotations = pd.read_csv(base_dir / "annotations" /
"all_annotations.csv")
class_count = all_annotations.groupby("split").apply(
lambda group: group.label.value_counts())
#%% Train model
margin = 0.05
k_shot = 4
cache = base_dir / "cache"
datasets = {
split: all_annotations.loc[lambda df: df.split == split].pipe(
ToKShotDataset(k_shot=k_shot,
preprocessing=data_augmentation,
cache=str(cache / split),
reset_cache=False))
for split in set(all_annotations.split)
}
batch_size = 64
encoder.trainable = False
optimizer = Adam(lr=1e-4)
model.compile(
optimizer=optimizer,
loss=class_consistency_loss,
metrics=[
accuracy(margin),
ClippedBinaryCrossentropy(),
MaxBinaryCrossentropy(),
StdBinaryCrossentropy(),
same_image_score,
classification_accuracy(),
],
)
model.fit(
datasets["train"].batch(batch_size).repeat(),
steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150,
validation_data=datasets["val"].batch(batch_size).repeat(),
validation_steps=max(
len(class_count["val"]) * k_shot // batch_size, 100),
initial_epoch=0,
epochs=3,
callbacks=callbacks,
)
encoder.trainable = True
optimizer = Adam(lr=1e-5)
model.compile(
optimizer=optimizer,
loss=class_consistency_loss,
metrics=[
accuracy(margin),
ClippedBinaryCrossentropy(),
MaxBinaryCrossentropy(),
StdBinaryCrossentropy(),
same_image_score,
classification_accuracy(),
],
)
model.fit(
datasets["train"].batch(batch_size).repeat(),
steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150,
validation_data=datasets["val"].batch(batch_size).repeat(),
validation_steps=max(
len(class_count["val"]) * k_shot // batch_size, 100),
initial_epoch=3,
epochs=30,
callbacks=callbacks,
)
#%% Evaluate on test set. Each batch is a k_shot, n_way=batch_size / k_shot task
model.load_weights(str(base_dir / "best_loss.h5"))
model.evaluate(datasets["test"].batch(batch_size).repeat(),
steps=max(
len(class_count["test"]) * k_shot // batch_size, 100))
#%% Export artifacts
classifier = Sequential([encoder, Classification(support_layer.kernel)])
tf.saved_model.save(classifier,
"siamese_nets_classifier/1",
signatures={"preprocessing": preprocessing})
val_cache = val_ds.cache().prefetch(buffer_size=auto_tune)
# 모델을 학습 및 로드/세이브한다.
# MobileNetV2_model, MobileNetV2_history = load_model(
# 'MobileNetV2.h5',
# applications.MobileNetV2(input_shape=data_shape, include_top=False),
# train_cache,
# val_cache,
# data_shape,
# num_classes,
# batch_size,
# epochs
# )
MobileNet_model, MobileNet_history = load_model(
'MobileNet.h5',
applications.MobileNet(input_shape=data_shape, include_top=False),
train_cache, val_cache, data_shape, num_classes, batch_size, epochs)
# NASNetMobile_model, NASNetMobile_history = load_model(
# 'NASNetMobile.h5',
# applications.NASNetMobile(input_shape=data_shape, include_top=False),
# train_cache,
# val_cache,
# data_shape,
# num_classes,
# batch_size,
# epochs
# )
# EfficientNetB0_model, EfficientNetB0_history = load_model(
# 'EfficientNetB0.h5',
# applications.EfficientNetB0(input_shape=data_shape, include_top=False),
# train_cache,
inputs = keras.Input(shape=INPUT_IMAGE_SIZE + [3])
x = layers.Rescaling(scale=1. / 255)(inputs)
x = layers.RandomCrop(160, 160)(x)
x = layers.RandomFlip(mode="horizontal")(x)
# We load the pretrained network, remove the top layers, and
# freeze the pre-trained weights.
if pretrained == 'VGG16':
pt_model = applications.VGG16(weights='imagenet',
include_top=False,
input_tensor=x)
elif pretrained == 'MobileNet':
pt_model = applications.MobileNet(weights='imagenet',
include_top=False,
input_tensor=x)
else:
assert 0, "Unknown model: " + pretrained
pt_name = pt_model.name
print('Using "{}" pre-trained model with {} layers'.format(
pt_name, len(pt_model.layers)))
for layer in pt_model.layers:
layer.trainable = False
# We then stack our own, randomly initialized layers on top of the
# pre-trained network.
x = layers.Flatten()(pt_model.output)
def train(base_dir):
#%% Init model
encoder = keras_applications.MobileNet(input_shape=(224, 224, 3), include_top=False, pooling="avg")
support_layer = CentroidsMatrix(
kernel={
"name": "MixedNorms",
"init": {
"norms": [
lambda x: 1 - tf.nn.l2_normalize(x[0]) * tf.nn.l2_normalize(x[1]),
lambda x: tf.math.abs(x[0] - x[1]),
lambda x: tf.nn.softmax(tf.math.abs(x[0] - x[1])),
lambda x: tf.square(x[0] - x[1]),
],
"use_bias": True,
},
},
activation="linear",
)
#%% Init training
callbacks = [
TensorBoard(base_dir, write_images=True, histogram_freq=1),
ModelCheckpoint(str(base_dir / "best_loss.h5"), save_best_only=True),
ReduceLROnPlateau(),
]
#%% Init data
@tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8),))
def preprocessing(input_tensor):
output_tensor = tf.cast(input_tensor, dtype=tf.float32)
output_tensor = tf.image.resize_with_pad(output_tensor, target_height=224, target_width=224)
output_tensor = keras_applications.mobilenet.preprocess_input(output_tensor, data_format="channels_last")
return output_tensor
@tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3], dtype=tf.float32),))
def data_augmentation(input_tensor):
output_tensor = tf.image.random_flip_left_right(input_tensor)
output_tensor = tf.image.random_flip_up_down(output_tensor)
output_tensor = tf.image.random_brightness(output_tensor, max_delta=0.25)
return output_tensor
all_annotations = pd.read_csv(base_dir / "annotations" / "all_annotations.csv").assign(
label_code=lambda df: df.label.astype("category").cat.codes
)
class_count = all_annotations.groupby("split").apply(lambda group: group.label.value_counts())
#%% Train model
k_shot = 4
cache = base_dir / "cache"
datasets = all_annotations.groupby("split").apply(
lambda group: (
ToKShotDataset(
k_shot=k_shot,
preprocessing=compose(preprocessing, data_augmentation),
cache=str(cache / group.name),
reset_cache=True,
dataset_mode="with_cache",
label_column="label_code",
)(group)
)
)
y_true = Input(shape=(None,), name="y_true")
output = support_layer([encoder.output, y_true])
model = Model([encoder.inputs, y_true], output)
batch_size = 64
batched_datasets = datasets.map(
lambda dataset: dataset.batch(batch_size, drop_remainder=True)
.map(lambda x, y: (x, get_dummies(y)[0]), num_parallel_calls=tf.data.experimental.AUTOTUNE)
.map(lambda x, y: ((x, y), y), num_parallel_calls=tf.data.experimental.AUTOTUNE)
.repeat()
)
encoder.trainable = False
optimizer = Adam(lr=1e-4)
model.compile(
optimizer=optimizer, loss="binary_crossentropy", metrics=["categorical_accuracy", "categorical_crossentropy"]
)
model.fit(
datasets["train"].batch(batch_size).repeat(),
steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150,
validation_data=datasets["val"].batch(batch_size).repeat(),
validation_steps=max(len(class_count["val"]) * k_shot // batch_size, 100),
initial_epoch=0,
epochs=3,
callbacks=callbacks,
)
encoder.trainable = True
optimizer = Adam(lr=1e-5)
model.compile(
optimizer=optimizer, loss="binary_crossentropy", metrics=["categorical_accuracy", "categorical_crossentropy"]
)
model.fit(
datasets["train"].batch(batch_size).repeat(),
steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150,
validation_data=datasets["val"].batch(batch_size).repeat(),
validation_steps=max(len(class_count["val"]) * k_shot // batch_size, 100),
initial_epoch=3,
epochs=10,
callbacks=callbacks,
)
#%% Evaluate on test set. Each batch is a k_shot, n_way=batch_size / k_shot task
model.load_weights(str(base_dir / "best_loss.h5"))
model.evaluate(batched_datasets["test"], steps=max(len(class_count["test"]) * k_shot // batch_size, 100))
print("\nValidation Data Set")
val_generator = ImageDataGenerator(preprocessing_function=preprocess_input)
val_flow = val_generator.flow_from_directory(val_path,
target_size=(HEIGHT, WIDTH),
batch_size=BATCH_SIZE)
#Test DataSet Generator with Augmentation
print("\nTest Data Set")
test_generator = ImageDataGenerator(preprocessing_function=preprocess_input)
test_flow = test_generator.flow_from_directory(test_path,
target_size=(HEIGHT, WIDTH),
batch_size=BATCH_SIZE)
# Initialize MobileNet with transfer learning
base_model = applications.MobileNet(weights='imagenet',
include_top=False,
input_shape=(WIDTH, HEIGHT, 3))
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# and a dense layer
x = Dense(1024, activation='relu')(x)
predictions = Dense(len(train_flow.class_indices), activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional MobileNet layers
