def test_should_return_centroids_of_input_tensors_according_to_their_class( ): input_tensors = tf.cast( tf.tile(tf.expand_dims(tf.range(5), 1), [1, 4]), tf.float32) labels_one_hot = get_dummies(tf.constant([0, 0, 0, 1, 1]))[0] support_tensors = CentroidsMatrix( kernel=sentinel.kernel).build_support_set( [input_tensors, labels_one_hot]) np.testing.assert_array_equal([[1, 1, 1, 1], [3.5, 3.5, 3.5, 3.5]], support_tensors.numpy())
def test_loss_should_equal_literal_calculation_for_semi_hard_mining(): np.random.seed(0) margin = 1 batch_size = 64 distance_matrix = np.random.rand(batch_size, batch_size) distance_matrix = distance_matrix + distance_matrix.T # distance matrix is symmetric labels = np.random.choice(list(range(4)), batch_size) # compute loss with for loop loss = [] for anchor_index in range(batch_size): for positive_index in range(batch_size): if positive_index == anchor_index: continue if labels[positive_index] == labels[anchor_index]: positive_distance = distance_matrix[anchor_index, positive_index] negative_distances = [] for negative_index in range(batch_size): if labels[negative_index] == labels[anchor_index]: continue negative_distances += [ distance_matrix[anchor_index, negative_index] ] negative_distances = np.array(negative_distances) if np.any(negative_distances - positive_distance > 0): negative_distance = np.min( negative_distances[negative_distances - positive_distance > 0]) else: print(anchor_index) print(positive_index) negative_distance = np.max(negative_distances) loss += [ positive_distance - negative_distance + margin ] np_loss = np.mean(loss) # assert value is equal y_true = get_dummies(labels)[0] y_pred = tf.convert_to_tensor(distance_matrix, dtype=tf.float32) tf_loss = TripletLoss(margin)(y_true, y_pred) np.testing.assert_almost_equal(np_loss, tf_loss, decimal=5)
{"name": "", "projector": []}, {"name": "_l2_normalize", "projector": [Lambda(lambda x: tf.math.l2_normalize(x, axis=1))]}, {"name": "_dense_10", "projector": [Dense(10)]}, {"name": "_dense_128", "projector": [Dense(128)]}, ] for experiment, projector in itertools.product(experiments, projectors): pprint(experiment) pprint(projector) for i in range(10): encoder.load_weights(str(output_dir / "initial_encoder.h5")) model = Sequential([encoder, *projector["projector"], GramMatrix(kernel=experiment["kernel"])]) model.compile( optimizer="adam", loss=experiment["loss"], metrics=experiment["metrics"], ) model.fit( train_dataset.map(lambda x, y: (tf.image.convert_image_dtype(x, tf.float32), get_dummies(y)[0])).repeat(), epochs=100, steps_per_epoch=train_steps, validation_data=val_dataset.map( lambda x, y: (tf.image.convert_image_dtype(x, tf.float32), get_dummies(y)[0]) ).repeat(), validation_steps=val_steps, callbacks=[TensorBoard(str(output_dir / f"{experiment['name']}{projector['name']}_{i}"))], ) results += [ { "experiment": experiment["name"], "projector": projector["name"], "iteration": i, **dict( zip(
return output_tensor def preprocessing(input_tensor): return tf.image.convert_image_dtype(input_tensor, dtype=tf.float32) #%% Build datasets k_shot = 8 n_way = 8 train_dataset = (tfds.load(name="cifar10", split="train[:90%]").shuffle( 50000 * 9 // 10).batch(n_way, drop_remainder=True).map( lambda annotation: ( tf.repeat(annotation["image"], k_shot, axis=0), tf.cast( get_dummies(tf.repeat(annotation["id"], k_shot))[0], tf.float32 ), ), num_parallel_calls=tf.data.experimental.AUTOTUNE, ).unbatch().map(lambda x, y: (preprocessing(data_augmentation(x)), y), num_parallel_calls=tf.data.experimental.AUTOTUNE).batch( k_shot * n_way)) val_dataset, test_dataset = (dataset.map( lambda x, y: (preprocessing(x), tf.one_hot(y, depth=10)), num_parallel_calls=tf.data.experimental.AUTOTUNE, ).batch(128) for dataset in tfds.load( name="cifar10", split=["train[90%:]", "test"], as_supervised=True)) train_steps = len([ _ for _ in train_dataset
def test_should_handle_multi_dimensional_tensor(): samples = tf.constant([[1, 2, 0, 2, 3, 0, 10, 10]]) one_hot, columns = get_dummies(samples) dummies_df = pd.get_dummies(samples.numpy().flatten())[columns.numpy()] np.testing.assert_array_equal(dummies_df.values, one_hot.numpy())
def test_should_handle_string_dtype(): samples = tf.constant(["a", "b", "b", "a", "c"]) one_hot, columns = get_dummies(samples) np.testing.assert_array_equal(pd.get_dummies(samples.numpy().flatten()).values, one_hot.numpy())
def train(base_dir): #%% Init model encoder = keras_applications.MobileNet(input_shape=(224, 224, 3), include_top=False, pooling="avg") support_layer = CentroidsMatrix( kernel={ "name": "MixedNorms", "init": { "norms": [ lambda x: 1 - tf.nn.l2_normalize(x[0]) * tf.nn.l2_normalize(x[1]), lambda x: tf.math.abs(x[0] - x[1]), lambda x: tf.nn.softmax(tf.math.abs(x[0] - x[1])), lambda x: tf.square(x[0] - x[1]), ], "use_bias": True, }, }, activation="linear", ) #%% Init training callbacks = [ TensorBoard(base_dir, write_images=True, histogram_freq=1), ModelCheckpoint(str(base_dir / "best_loss.h5"), save_best_only=True), ReduceLROnPlateau(), ] #%% Init data @tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8),)) def preprocessing(input_tensor): output_tensor = tf.cast(input_tensor, dtype=tf.float32) output_tensor = tf.image.resize_with_pad(output_tensor, target_height=224, target_width=224) output_tensor = keras_applications.mobilenet.preprocess_input(output_tensor, data_format="channels_last") return output_tensor @tf.function(input_signature=(tf.TensorSpec(shape=[None, None, 3], dtype=tf.float32),)) def data_augmentation(input_tensor): output_tensor = tf.image.random_flip_left_right(input_tensor) output_tensor = tf.image.random_flip_up_down(output_tensor) output_tensor = tf.image.random_brightness(output_tensor, max_delta=0.25) return output_tensor all_annotations = pd.read_csv(base_dir / "annotations" / "all_annotations.csv").assign( label_code=lambda df: df.label.astype("category").cat.codes ) class_count = all_annotations.groupby("split").apply(lambda group: group.label.value_counts()) #%% Train model k_shot = 4 cache = base_dir / "cache" datasets = all_annotations.groupby("split").apply( lambda group: ( ToKShotDataset( k_shot=k_shot, preprocessing=compose(preprocessing, data_augmentation), cache=str(cache / group.name), reset_cache=True, dataset_mode="with_cache", label_column="label_code", )(group) ) ) y_true = Input(shape=(None,), name="y_true") output = support_layer([encoder.output, y_true]) model = Model([encoder.inputs, y_true], output) batch_size = 64 batched_datasets = datasets.map( lambda dataset: dataset.batch(batch_size, drop_remainder=True) .map(lambda x, y: (x, get_dummies(y)[0]), num_parallel_calls=tf.data.experimental.AUTOTUNE) .map(lambda x, y: ((x, y), y), num_parallel_calls=tf.data.experimental.AUTOTUNE) .repeat() ) encoder.trainable = False optimizer = Adam(lr=1e-4) model.compile( optimizer=optimizer, loss="binary_crossentropy", metrics=["categorical_accuracy", "categorical_crossentropy"] ) model.fit( datasets["train"].batch(batch_size).repeat(), steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150, validation_data=datasets["val"].batch(batch_size).repeat(), validation_steps=max(len(class_count["val"]) * k_shot // batch_size, 100), initial_epoch=0, epochs=3, callbacks=callbacks, ) encoder.trainable = True optimizer = Adam(lr=1e-5) model.compile( optimizer=optimizer, loss="binary_crossentropy", metrics=["categorical_accuracy", "categorical_crossentropy"] ) model.fit( datasets["train"].batch(batch_size).repeat(), steps_per_epoch=len(class_count["train"]) * k_shot // batch_size * 150, validation_data=datasets["val"].batch(batch_size).repeat(), validation_steps=max(len(class_count["val"]) * k_shot // batch_size, 100), initial_epoch=3, epochs=10, callbacks=callbacks, ) #%% Evaluate on test set. Each batch is a k_shot, n_way=batch_size / k_shot task model.load_weights(str(base_dir / "best_loss.h5")) model.evaluate(batched_datasets["test"], steps=max(len(class_count["test"]) * k_shot // batch_size, 100))
loss=tfa.losses.TripletSemiHardLoss(), ) encoder.fit(train_dataset.map(lambda x, y: (preprocessing(x), y)), epochs=5, callbacks=[TensorBoard("tfa_loss")]) encoder.evaluate(test_dataset.map(lambda x, y: (preprocessing(x), y))) results = encoder.predict(test_dataset.map(lambda x, y: (preprocessing(x), y))) np.savetxt("tfa_embeddings.tsv", results, delimiter="\t") #%% Train with keras_fsl triplet loss encoder.load_weights("initial_encoder.h5") model.compile(optimizer=tf.keras.optimizers.Adam(0.001), loss=triplet_loss(), metrics=[classification_accuracy(ascending=True)]) model.fit( train_dataset.map(lambda x, y: (preprocessing(x), get_dummies(y)[0])), epochs=5, callbacks=[TensorBoard("keras_fsl_loss")]) model.evaluate( test_dataset.map(lambda x, y: (preprocessing(x), get_dummies(y)[0]))) results = encoder.predict( test_dataset.map(lambda x, y: (preprocessing(x), get_dummies(y)[0]))) np.savetxt("keras_fsl_embeddings.tsv", results, delimiter="\t") #%% Try with l1 norm support_layer.kernel = Lambda( lambda inputs: tf.math.reduce_sum(tf.abs(inputs[0] - inputs[1]), axis=1)) encoder.load_weights("initial_encoder.h5") model.compile(optimizer=tf.keras.optimizers.Adam(0.001), loss=triplet_loss(), metrics=[classification_accuracy(ascending=True)])