def test_should_return_centroids_of_input_tensors_according_to_their_class(
):
input_tensors = tf.cast(
tf.tile(tf.expand_dims(tf.range(5), 1), [1, 4]), tf.float32)
labels_one_hot = get_dummies(tf.constant([0, 0, 0, 1, 1]))[0]
support_tensors = CentroidsMatrix(
kernel=sentinel.kernel).build_support_set(
[input_tensors, labels_one_hot])
np.testing.assert_array_equal([[1, 1, 1, 1], [3.5, 3.5, 3.5, 3.5]],
support_tensors.numpy())
def test_loss_should_equal_literal_calculation_for_semi_hard_mining():
np.random.seed(0)
margin = 1
batch_size = 64
distance_matrix = np.random.rand(batch_size, batch_size)
distance_matrix = distance_matrix + distance_matrix.T # distance matrix is symmetric
labels = np.random.choice(list(range(4)), batch_size)
# compute loss with for loop
loss = []
for anchor_index in range(batch_size):
for positive_index in range(batch_size):
if positive_index == anchor_index:
continue
if labels[positive_index] == labels[anchor_index]:
positive_distance = distance_matrix[anchor_index,
positive_index]
negative_distances = []
for negative_index in range(batch_size):
if labels[negative_index] == labels[anchor_index]:
continue
negative_distances += [
distance_matrix[anchor_index, negative_index]
]
negative_distances = np.array(negative_distances)
if np.any(negative_distances - positive_distance > 0):
negative_distance = np.min(
negative_distances[negative_distances -
positive_distance > 0])
else:
print(anchor_index)
print(positive_index)
negative_distance = np.max(negative_distances)
loss += [
positive_distance - negative_distance + margin
]
np_loss = np.mean(loss)
# assert value is equal
y_true = get_dummies(labels)[0]
y_pred = tf.convert_to_tensor(distance_matrix, dtype=tf.float32)
tf_loss = TripletLoss(margin)(y_true, y_pred)
np.testing.assert_almost_equal(np_loss, tf_loss, decimal=5)
{"name": "", "projector": []},
{"name": "_l2_normalize", "projector": [Lambda(lambda x: tf.math.l2_normalize(x, axis=1))]},
{"name": "_dense_10", "projector": [Dense(10)]},
{"name": "_dense_128", "projector": [Dense(128)]},
]
for experiment, projector in itertools.product(experiments, projectors):
pprint(experiment)
pprint(projector)
for i in range(10):
encoder.load_weights(str(output_dir / "initial_encoder.h5"))
model = Sequential([encoder, *projector["projector"], GramMatrix(kernel=experiment["kernel"])])
model.compile(
optimizer="adam", loss=experiment["loss"], metrics=experiment["metrics"],
)
model.fit(
train_dataset.map(lambda x, y: (tf.image.convert_image_dtype(x, tf.float32), get_dummies(y)[0])).repeat(),
epochs=100,
steps_per_epoch=train_steps,
validation_data=val_dataset.map(
lambda x, y: (tf.image.convert_image_dtype(x, tf.float32), get_dummies(y)[0])
).repeat(),
validation_steps=val_steps,
callbacks=[TensorBoard(str(output_dir / f"{experiment['name']}{projector['name']}_{i}"))],
)
results += [
{
"experiment": experiment["name"],
"projector": projector["name"],
"iteration": i,
**dict(
zip(
return output_tensor
def preprocessing(input_tensor):
return tf.image.convert_image_dtype(input_tensor, dtype=tf.float32)
#%% Build datasets
k_shot = 8
n_way = 8
train_dataset = (tfds.load(name="cifar10", split="train[:90%]").shuffle(
50000 * 9 // 10).batch(n_way, drop_remainder=True).map(
lambda annotation: (
tf.repeat(annotation["image"], k_shot, axis=0),
tf.cast(
get_dummies(tf.repeat(annotation["id"], k_shot))[0], tf.float32
),
),
num_parallel_calls=tf.data.experimental.AUTOTUNE,
).unbatch().map(lambda x, y: (preprocessing(data_augmentation(x)), y),
num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(
k_shot * n_way))
val_dataset, test_dataset = (dataset.map(
lambda x, y: (preprocessing(x), tf.one_hot(y, depth=10)),
num_parallel_calls=tf.data.experimental.AUTOTUNE,
).batch(128) for dataset in tfds.load(
name="cifar10", split=["train[90%:]", "test"], as_supervised=True))
train_steps = len([
_ for _ in train_dataset
def test_should_handle_multi_dimensional_tensor():
samples = tf.constant([[1, 2, 0, 2, 3, 0, 10, 10]])
one_hot, columns = get_dummies(samples)
dummies_df = pd.get_dummies(samples.numpy().flatten())[columns.numpy()]
np.testing.assert_array_equal(dummies_df.values, one_hot.numpy())
def test_should_handle_string_dtype():
samples = tf.constant(["a", "b", "b", "a", "c"])
one_hot, columns = get_dummies(samples)
np.testing.assert_array_equal(pd.get_dummies(samples.numpy().flatten()).values, one_hot.numpy())
def train(base_dir):
#%% Init model
encoder = keras_applications.MobileNet(input_shape=(224, 224, 3), include_top=False, pooling="avg")
support_layer = CentroidsMatrix(
kernel={
"name": "MixedNorms",
"init": {
"norms": [
lambda x: 1 - tf.nn.l2_normalize(x[0]) * tf.nn.l2_normalize(x[1]),
lambda x: tf.math.abs(x[0] - x[1]),
lambda x: tf.nn.softmax(tf.math.abs(x[0] - x[1])),
lambda x: tf.square(x[0] - x[1]),
],
"use_bias": True,
},
},
activation="linear",
)
#%% Init training
callbacks = [
TensorBoard(base_dir, write_images=True, histogram_freq=1),
ModelCheckpoint(str(base_dir / "best_loss.h5"), save_best_only=True),
ReduceLROnPlateau(),
]
#%% Init data
@tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8),))
def preprocessing(input_tensor):
output_tensor = tf.cast(input_tensor, dtype=tf.float32)
output_tensor = tf.image.resize_with_pad(output_tensor, target_height=224, target_width=224)
output_tensor = keras_applications.mobilenet.preprocess_input(output_tensor, data_format="channels_last")
return output_tensor
@tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3], dtype=tf.float32),))
def data_augmentation(input_tensor):
output_tensor = tf.image.random_flip_left_right(input_tensor)
output_tensor = tf.image.random_flip_up_down(output_tensor)
output_tensor = tf.image.random_brightness(output_tensor, max_delta=0.25)
return output_tensor
all_annotations = pd.read_csv(base_dir / "annotations" / "all_annotations.csv").assign(
label_code=lambda df: df.label.astype("category").cat.codes
)
class_count = all_annotations.groupby("split").apply(lambda group: group.label.value_counts())
#%% Train model
k_shot = 4
cache = base_dir / "cache"
datasets = all_annotations.groupby("split").apply(
lambda group: (
ToKShotDataset(
k_shot=k_shot,
preprocessing=compose(preprocessing, data_augmentation),
cache=str(cache / group.name),
reset_cache=True,
dataset_mode="with_cache",
label_column="label_code",
)(group)
)
)
y_true = Input(shape=(None,), name="y_true")
output = support_layer([encoder.output, y_true])
model = Model([encoder.inputs, y_true], output)
batch_size = 64
batched_datasets = datasets.map(
lambda dataset: dataset.batch(batch_size, drop_remainder=True)
.map(lambda x, y: (x, get_dummies(y)[0]), num_parallel_calls=tf.data.experimental.AUTOTUNE)
.map(lambda x, y: ((x, y), y), num_parallel_calls=tf.data.experimental.AUTOTUNE)
.repeat()
)
encoder.trainable = False
optimizer = Adam(lr=1e-4)
model.compile(
optimizer=optimizer, loss="binary_crossentropy", metrics=["categorical_accuracy", "categorical_crossentropy"]
)
model.fit(
datasets["train"].batch(batch_size).repeat(),
steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150,
validation_data=datasets["val"].batch(batch_size).repeat(),
validation_steps=max(len(class_count["val"]) * k_shot // batch_size, 100),
initial_epoch=0,
epochs=3,
callbacks=callbacks,
)
encoder.trainable = True
optimizer = Adam(lr=1e-5)
model.compile(
optimizer=optimizer, loss="binary_crossentropy", metrics=["categorical_accuracy", "categorical_crossentropy"]
)
model.fit(
datasets["train"].batch(batch_size).repeat(),
steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150,
validation_data=datasets["val"].batch(batch_size).repeat(),
validation_steps=max(len(class_count["val"]) * k_shot // batch_size, 100),
initial_epoch=3,
epochs=10,
callbacks=callbacks,
)
#%% Evaluate on test set. Each batch is a k_shot, n_way=batch_size / k_shot task
model.load_weights(str(base_dir / "best_loss.h5"))
model.evaluate(batched_datasets["test"], steps=max(len(class_count["test"]) * k_shot // batch_size, 100))
loss=tfa.losses.TripletSemiHardLoss(),
)
encoder.fit(train_dataset.map(lambda x, y: (preprocessing(x), y)),
epochs=5,
callbacks=[TensorBoard("tfa_loss")])
encoder.evaluate(test_dataset.map(lambda x, y: (preprocessing(x), y)))
results = encoder.predict(test_dataset.map(lambda x, y: (preprocessing(x), y)))
np.savetxt("tfa_embeddings.tsv", results, delimiter="\t")
#%% Train with keras_fsl triplet loss
encoder.load_weights("initial_encoder.h5")
model.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss=triplet_loss(),
metrics=[classification_accuracy(ascending=True)])
model.fit(
train_dataset.map(lambda x, y: (preprocessing(x), get_dummies(y)[0])),
epochs=5,
callbacks=[TensorBoard("keras_fsl_loss")])
model.evaluate(
test_dataset.map(lambda x, y: (preprocessing(x), get_dummies(y)[0])))
results = encoder.predict(
test_dataset.map(lambda x, y: (preprocessing(x), get_dummies(y)[0])))
np.savetxt("keras_fsl_embeddings.tsv", results, delimiter="\t")
#%% Try with l1 norm
support_layer.kernel = Lambda(
lambda inputs: tf.math.reduce_sum(tf.abs(inputs[0] - inputs[1]), axis=1))
encoder.load_weights("initial_encoder.h5")
model.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss=triplet_loss(),
metrics=[classification_accuracy(ascending=True)])