"loss": class_consistency_loss,
"metrics": [classification_accuracy(ascending=False), class_consistency_loss, BinaryCrossentropy()],
},
]
projectors = [
{"name": "", "projector": []},
{"name": "_l2_normalize", "projector": [Lambda(lambda x: tf.math.l2_normalize(x, axis=1))]},
{"name": "_dense_10", "projector": [Dense(10)]},
{"name": "_dense_128", "projector": [Dense(128)]},
]
for experiment, projector in itertools.product(experiments, projectors):
pprint(experiment)
pprint(projector)
for i in range(10):
encoder.load_weights(str(output_dir / "initial_encoder.h5"))
model = Sequential([encoder, *projector["projector"], GramMatrix(kernel=experiment["kernel"])])
model.compile(
optimizer="adam", loss=experiment["loss"], metrics=experiment["metrics"],
)
model.fit(
train_dataset.map(lambda x, y: (tf.image.convert_image_dtype(x, tf.float32), get_dummies(y)[0])).repeat(),
epochs=100,
steps_per_epoch=train_steps,
validation_data=val_dataset.map(
lambda x, y: (tf.image.convert_image_dtype(x, tf.float32), get_dummies(y)[0])
).repeat(),
validation_steps=val_steps,
callbacks=[TensorBoard(str(output_dir / f"{experiment['name']}{projector['name']}_{i}"))],
)
results += [
{
#%% Train
experiments = [
{
"name": "binary_crossentropy",
"loss": ClippedBinaryCrossentropy(upper=0.75)
},
{
"name": "class_consistency",
"loss": ClassConsistencyLoss()
},
]
for experiment in experiments:
pprint(experiment)
encoder.load_weights(str(output_dir / "initial_encoder.h5"))
model = Sequential([encoder, GramMatrix(kernel="LearntNorms")])
model.compile(
optimizer="adam",
loss=experiment["loss"],
metrics=[classification_accuracy(ascending=False)],
)
model.fit(
train_dataset.repeat(),
epochs=10,
steps_per_epoch=train_steps,
validation_data=val_dataset.repeat(),
validation_steps=val_steps,
callbacks=[
TensorBoard(str(output_dir / experiment["name"])),
EarlyStopping(patience=10)
],
def train(base_dir):
#%% Init model
encoder = keras_applications.MobileNet(input_shape=(224, 224, 3),
include_top=False,
pooling="avg")
support_layer = GramMatrix(kernel={
"name": "MixedNorms",
"init": {
"norms": [
lambda x: 1 - tf.nn.l2_normalize(x[0]) * tf.nn.l2_normalize(x[
1]),
lambda x: tf.math.abs(x[0] - x[1]),
lambda x: tf.nn.softmax(tf.math.abs(x[0] - x[1])),
lambda x: tf.square(x[0] - x[1]),
],
"use_bias":
True,
},
}, )
model = Sequential([encoder, support_layer])
#%% Init training
callbacks = [
TensorBoard(base_dir, write_images=True, histogram_freq=1),
ModelCheckpoint(str(base_dir / "best_loss.h5"), save_best_only=True),
ReduceLROnPlateau(),
]
#%% Init data
@tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3],
dtype=tf.uint8), ))
def preprocessing(input_tensor):
output_tensor = tf.cast(input_tensor, dtype=tf.float32)
output_tensor = tf.image.resize_with_pad(output_tensor,
target_height=224,
target_width=224)
output_tensor = keras_applications.mobilenet.preprocess_input(
output_tensor, data_format="channels_last")
return output_tensor
@tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3],
dtype=tf.float32), ))
def data_augmentation(input_tensor):
output_tensor = tf.image.random_flip_left_right(input_tensor)
output_tensor = tf.image.random_flip_up_down(output_tensor)
output_tensor = tf.image.random_brightness(output_tensor,
max_delta=0.25)
return preprocessing(output_tensor)
all_annotations = pd.read_csv(base_dir / "annotations" /
"all_annotations.csv")
class_count = all_annotations.groupby("split").apply(
lambda group: group.label.value_counts())
#%% Train model
margin = 0.05
k_shot = 4
cache = base_dir / "cache"
datasets = {
split: all_annotations.loc[lambda df: df.split == split].pipe(
ToKShotDataset(k_shot=k_shot,
preprocessing=data_augmentation,
cache=str(cache / split),
reset_cache=False))
for split in set(all_annotations.split)
}
batch_size = 64
encoder.trainable = False
optimizer = Adam(lr=1e-4)
model.compile(
optimizer=optimizer,
loss=class_consistency_loss,
metrics=[
accuracy(margin),
ClippedBinaryCrossentropy(),
MaxBinaryCrossentropy(),
StdBinaryCrossentropy(),
same_image_score,
classification_accuracy(),
],
)
model.fit(
datasets["train"].batch(batch_size).repeat(),
steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150,
validation_data=datasets["val"].batch(batch_size).repeat(),
validation_steps=max(
len(class_count["val"]) * k_shot // batch_size, 100),
initial_epoch=0,
epochs=3,
callbacks=callbacks,
)
encoder.trainable = True
optimizer = Adam(lr=1e-5)
model.compile(
optimizer=optimizer,
loss=class_consistency_loss,
metrics=[
accuracy(margin),
ClippedBinaryCrossentropy(),
MaxBinaryCrossentropy(),
StdBinaryCrossentropy(),
same_image_score,
classification_accuracy(),
],
)
model.fit(
datasets["train"].batch(batch_size).repeat(),
steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150,
validation_data=datasets["val"].batch(batch_size).repeat(),
validation_steps=max(
len(class_count["val"]) * k_shot // batch_size, 100),
initial_epoch=3,
epochs=30,
callbacks=callbacks,
)
#%% Evaluate on test set. Each batch is a k_shot, n_way=batch_size / k_shot task
model.load_weights(str(base_dir / "best_loss.h5"))
model.evaluate(datasets["test"].batch(batch_size).repeat(),
steps=max(
len(class_count["test"]) * k_shot // batch_size, 100))
#%% Export artifacts
classifier = Sequential([encoder, Classification(support_layer.kernel)])
tf.saved_model.save(classifier,
"siamese_nets_classifier/1",
signatures={"preprocessing": preprocessing})
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = key_to_use
projector.visualize_embeddings(output_dir, config)
#%% Evaluate model
scores = model.evaluate(X_test, y_test, verbose=1)
print(f"Accuracy: {scores[1]:.2%}")
#%% Train with binary crossentropy and gram matrix
accuracies = []
for i in range(1, 21):
kernel = Lambda(
lambda inputs: tf.reduce_sum(inputs[0] * inputs[1], axis=1))
model = Sequential([BasicCNN((32, 32, 3), i), GramMatrix(kernel)])
model.summary()
model.compile(
optimizer="adam",
loss=binary_crossentropy(),
metrics=[mean_score_classification_loss, min_eigenvalue],
)
model.fit(X_train, y_train, validation_split=0.2, epochs=20, batch_size=32)
embeddings = model.layers[0].predict(X_train)
classifier = Sequential([model.layers[0], Classification(kernel)])
classifier.layers[1].set_support_set(embeddings, y_train)
classifier.compile(loss="binary_crossentropy", optimizer="adam")
classifier.evaluate(X_test, y_test, verbose=1)
y_pred = classifier.predict(X_test, verbose=1)
confusion_matrix = pd.crosstab(
kernel_size=2,
padding="same",
activation="relu",
input_shape=(28, 28, 1)),
MaxPooling2D(pool_size=2),
Dropout(0.3),
Conv2D(filters=32, kernel_size=2, padding="same", activation="relu"),
MaxPooling2D(pool_size=2),
Dropout(0.3),
Flatten(),
Dense(256, activation=None), # No activation on final dense layer
Lambda(
lambda x: tf.math.l2_normalize(x, axis=1)), # L2 normalize embeddings
])
encoder.save_weights("initial_encoder.h5")
support_layer = GramMatrix(kernel=Lambda(lambda inputs: tf.math.reduce_sum(
tf.square(inputs[0] - inputs[1]), axis=1)))
model = Sequential([encoder, support_layer])
#%% Build datasets
def preprocessing(input_tensor):
return tf.cast(input_tensor, tf.float32) / 255
train_dataset, test_dataset = tfds.load(name="mnist",
split=["train", "test"],
as_supervised=True)
train_dataset = train_dataset.shuffle(1024).batch(32)
test_dataset = test_dataset.batch(32)
#%% Save test labels for later visualization in projector https://projector.tensorflow.org/
cv = StratifiedKFold(5)
for train, test in cv.split(X, Y):
x_train, y_train = X.iloc[train], Y.iloc[train]
x_test, y_test = X.iloc[test], Y.iloc[test]
x_support, x_query, y_support, y_query = train_test_split(x_train,
y_train,
test_size=0.15,
stratify=y_train)
# create the model
input_shape = x_support.shape[1] # first shape is batch_size
# %% Training
encoder = Dense(73)
support_layer = GramMatrix("DenseSigmoid")
model = Sequential([encoder, support_layer])
model.compile(optimizer="Adam",
loss=BinaryCrossentropy(),
metrics=[classification_accuracy(), min_eigenvalue])
model.fit(x=x_support,
y=y_support,
validation_data=([x_query], [y_query]),
epochs=5)
# model = XGBClassifier(class_weights=[1,5])
# model.fit(x_train, y_train)ddd
y_pred = model.predict_proba(x_test)[:, 1]
print_results(y_test, y_pred)