def get_task():
meta_batch_vectors = list()
for meta_batch in range(self.meta_batch_size):
vectors = self.generate_all_vectors(latent_dim)
vectors = tf.reshape(tf.stack(vectors, axis=0),
(-1, latent_dim))
meta_batch_vectors.append(vectors)
meta_batch_vectors = tf.stack(meta_batch_vectors)
meta_batch_vectors = combine_first_two_axes(meta_batch_vectors)
images = get_images_from_vectors(meta_batch_vectors)
images = tf.image.resize(images, (84, 84))
images = tf.reshape(
images, (self.meta_batch_size, self.n *
(self.k + self.k_val_ml), *generated_image_shape))
train_ds = images[:, :self.n * self.k, ...]
train_indices = [
i // self.k + i % self.k * self.n
for i in range(self.n * self.k)
]
train_ds = tf.gather(train_ds, train_indices, axis=1)
train_ds = tf.reshape(
train_ds,
(self.meta_batch_size, self.n, self.k, *generated_image_shape))
val_ds = images[:, self.n * self.k:, ...]
val_ds = combine_first_two_axes(val_ds)
# random_num = tf.random.uniform(shape=(), minval=0, maxval=1)
# if random_num < 4:
# angles = tf.random.uniform(
# shape=(self.meta_batch_size * self.n * self.k_val_ml, ),
# minval=tf.constant(-np.pi),
# maxval=tf.constant(np.pi)
# )
# val_ds = tfa.image.rotate(val_ds, angles)
# else:
# val_ds = tf_image_translate(
# val_ds,
# tf.random.uniform((), -5, 5, dtype=tf.int32),
# tf.random.uniform((), -5, 5, dtype=tf.int32)
# )
val_ds = tf.reshape(val_ds,
(self.meta_batch_size, self.n * self.k_val_ml,
*generated_image_shape))
val_indices = [
i // self.k_val_ml + i % self.k_val_ml * self.n
for i in range(self.n * self.k_val_ml)
]
val_ds = tf.gather(val_ds, val_indices, axis=1)
val_ds = tf.reshape(val_ds,
(self.meta_batch_size, self.n, self.k_val_ml,
*generated_image_shape))
yield (train_ds, val_ds), (train_labels, val_labels)
def get_task_train_and_val_ds(self, task, labels):
train_ds, val_ds = tf.split(task, num_or_size_splits=2)
train_labels, val_labels = tf.split(labels, num_or_size_splits=2)
train_ds = combine_first_two_axes(tf.squeeze(train_ds, axis=0))
val_ds = combine_first_two_axes(tf.squeeze(val_ds, axis=0))
train_labels = combine_first_two_axes(tf.squeeze(train_labels, axis=0))
val_labels = combine_first_two_axes(tf.squeeze(val_labels, axis=0))
return train_ds, val_ds, train_labels, val_labels
def f(inputs):
train_ds, train_dom, val_ds, val_dom, train_labels, val_labels = inputs
train_ds = combine_first_two_axes(train_ds)
train_dom = combine_first_two_axes(train_dom)
val_ds = combine_first_two_axes(val_ds)
val_dom = combine_first_two_axes(val_dom)
self._initialize_eval_model()
for i in range(iterations):
self._train_model_for_eval(train_ds, train_dom, train_labels)
val_acc, val_loss = self._evaluate_model_for_eval(val_ds, val_dom, val_labels, training)
return val_acc, val_loss
def evaluate(self, iterations, iterations_to_load_from=None, seed=-1):
# self.load_model(iterations=51, acc='0.78')
# self.load_model()
# model = load_model('facenet_keras.h5')
model = load_model(
'/home/siavash/PycharmProjects/facenet/model/facenet_keras.h5')
print(model.inputs)
print(model.outputs)
model.trainable = False
self.model = model
self.model.summary()
# dense = tf.keras.layers.Dense(self.n, activation='softmax')(model.output)
# self.model = tf.keras.models.Model(inputs=model.inputs, outputs=dense)
self.test_dataset = self.get_test_dataset(seed)
task_number = 0
accs = list()
for (train_ds, val_ds), (train_labels,
val_labels) in self.test_dataset:
task_number += 1
if task_number % (self.number_of_tasks_test // 20) == 0:
print(f'{task_number} finished.')
train_labels = combine_first_two_axes(train_labels)
val_labels = combine_first_two_axes(val_labels)
train_labels = tf.math.argmax(train_labels, axis=1)
val_labels = tf.math.argmax(val_labels, axis=1)
train_ds = tf.squeeze(train_ds, axis=0)
train_ds = combine_first_two_axes(train_ds)
encodings = self.model.predict(train_ds)
from datetime import datetime
begin = datetime.now()
val_ds = tf.squeeze(val_ds, axis=0)
val_ds = combine_first_two_axes(val_ds)
val_encodings = self.model.predict(val_ds)
dists = self.euclidean_distance(val_encodings, encodings)
# predicted_labels = train_labels[tf.argmin(dists, axis=1)]
predicted_labels = train_labels.numpy()[tf.argmin(dists,
axis=1).numpy()]
end = datetime.now()
print(end - begin)
task_final_accuracy = np.mean(predicted_labels == val_labels)
accs.append(task_final_accuracy)
print(f'accuracy mean: {np.mean(accs)}')
print(f'accuracy std: {np.std(accs)}')
print(
f'final acc: {np.mean(accs)} +- {1.96 * np.std(accs) / np.sqrt(self.number_of_tasks_test)}'
)
return np.mean(accs)
def get_task():
meta_batch_vectors = list()
for meta_batch in range(self.meta_batch_size):
vectors = self.generate_all_vectors()
vectors = tf.reshape(tf.stack(vectors, axis=0),
(-1, self.latent_dim))
meta_batch_vectors.append(vectors)
meta_batch_vectors = tf.stack(meta_batch_vectors)
meta_batch_vectors = combine_first_two_axes(meta_batch_vectors)
images = self.get_images_from_vectors(meta_batch_vectors)
images = tf.image.resize(images, self.generated_image_shape[:2])
images = tf.reshape(
images,
(self.meta_batch_size, self.n *
(self.k + self.k_val_ml), *self.generated_image_shape))
train_ds = images[:, :self.n * self.k, ...]
train_ds = tf.gather(train_ds, train_indices, axis=1)
train_ds = tf.reshape(train_ds,
(self.meta_batch_size, self.n, self.k,
*self.generated_image_shape))
val_ds = images[:, self.n * self.k:, ...]
val_ds = combine_first_two_axes(val_ds)
# ================
val_imgs = list()
for i in range(val_ds.shape[0]):
val_image = val_ds[i, ...]
tx = tf.random.uniform((), -5, 5, dtype=tf.int32)
ty = tf.random.uniform((), -5, 5, dtype=tf.int32)
transforms = [1, 0, -tx, 0, 1, -ty, 0, 0]
val_image = tfa.image.transform(val_image, transforms,
'NEAREST')
val_imgs.append(val_image)
val_ds = tf.stack(val_imgs, axis=0)
# ================
val_ds = tf.reshape(val_ds,
(self.meta_batch_size, self.n * self.k_val_ml,
*self.generated_image_shape))
val_ds = tf.gather(val_ds, val_indices, axis=1)
val_ds = tf.reshape(val_ds,
(self.meta_batch_size, self.n, self.k_val_ml,
*self.generated_image_shape))
yield (train_ds, val_ds), (train_labels, val_labels)
def f(inputs):
train_ds, val_ds, train_labels, val_labels = inputs
train_ds = combine_first_two_axes(train_ds)
val_ds = combine_first_two_axes(val_ds)
self._initialize_eval_model()
for i in range(iterations):
self._train_model_for_eval(train_ds, train_labels)
val_acc, val_loss = self._evaluate_model_for_eval(val_ds, val_labels, training)
if settings.DEBUG:
tf.print()
tf.print(val_loss)
tf.print(val_acc)
tf.print()
return val_acc, val_loss
def get_task(resolution=(25, 25, 3), tf_seed=3, n=5):
# return (train_ds, val_ds), (train_labels, val_labels)
latent_dim = 512 # 100 for omniglot
generated_image_shape = resolution # (28, 28, 1) for omniglot
vectors, noise_vectors = generate_all_vectors_by_interpolation(latent_dim, tf_seed, n=n)
vectors = tf.reshape(tf.stack(vectors, axis=0), (-1, latent_dim))
images = gan_generator(vectors)['default']
images = tf.image.resize(images, generated_image_shape[:2])
train_ds = images[:n * k]
train_indices = [i // k + i % k * n for i in range(n * k)]
train_ds = tf.gather(train_ds, train_indices, axis=0)
train_ds = tf.reshape(train_ds, (n, k, *generated_image_shape))
val_ds = images[n * k:]
val_indices = [i // k_val_ml + i % k_val_ml * n for i in range(n * k_val_ml)]
val_ds = tf.gather(val_ds, val_indices, axis=0)
val_ds = tf.reshape(val_ds, (n, k_val_ml, *generated_image_shape))
val_ds = combine_first_two_axes(val_ds)
val_ds = tf.reshape(val_ds, (n, k_val_ml, *generated_image_shape))
noise_images = gan_generator(noise_vectors)['default']
noise_images = tf.image.resize(noise_images, generated_image_shape[:2])
train_labels = np.repeat(np.arange(n), k)
val_labels = np.repeat(np.arange(n), k_val_ml)
train_labels = tf.one_hot(train_labels, depth=n)
val_labels = tf.one_hot(val_labels, depth=n)
yield (train_ds, val_ds), (train_labels, val_labels), vectors, noise_vectors, noise_images
def f(inputs):
train_ds, val_ds, train_labels, val_labels = inputs
train_ds = combine_first_two_axes(train_ds)
val_ds = combine_first_two_axes(val_ds)
updated_model, inner_losses = self.inner_train_loop(train_ds, train_labels)
# TODO test what happens when training=False
updated_model_logits = updated_model(val_ds, training=True)
val_loss = self.outer_loss(val_labels, updated_model_logits, inner_losses)
predicted_class_labels = self.predict_class_labels_from_logits(updated_model_logits)
real_labels = self.convert_labels_to_real_labels(val_labels)
val_acc = tf.reduce_mean(tf.cast(tf.equal(predicted_class_labels, real_labels), tf.float32))
return val_acc, val_loss
def test_covering_all_classes_in_one_epoch(self, mocked_parse_function):
# Make a new database so that the number of classes are dividable by the number of meta batches * n.
mocked_parse_function.return_value = self.parse_function
ds = self.omniglot_database.get_supervised_meta_learning_dataset(
self.omniglot_database.train_folders,
n=5,
k=5,
k_validation=5,
meta_batch_size=4,
dtype=tf.string)
# Check for covering all classes and no duplicates
classes = set()
for (x, x_val), (y, y_val) in ds:
train_ds = combine_first_two_axes(x)
for i in range(train_ds.shape[0]):
class_instances = train_ds[i, ...]
class_instance_address = class_instances[0].numpy().decode(
'utf-8')
class_address = os.path.split(class_instance_address)[0]
self.assertNotIn(class_address, classes)
classes.add(class_address)
self.assertSetEqual(classes,
set(self.omniglot_database.train_folders.keys()))
def test_covering_all_classes_in_subsequent_epochs(self,
mocked_parse_function):
# This test might not pass because of the random selection of classes at the beginning of the class, but the
# chances are low. Specially if we increase the number of epochs, the chance of not covering classes will
# decrease. Make a new database so that the number of classes are dividable by the number of meta batches * n.
# This test should always fail with num_epochs = 1
num_epochs = 3
mocked_parse_function.return_value = self.parse_function
ds = self.omniglot_database.get_supervised_meta_learning_dataset(
self.omniglot_database.train_folders,
n=7,
k=5,
k_validation=5,
meta_batch_size=4,
seed=42,
dtype=tf.string)
# Check for covering all classes
classes = set()
for epoch in range(num_epochs):
for (x, x_val), (y, y_val) in ds:
train_ds = combine_first_two_axes(x)
for i in range(train_ds.shape[0]):
class_instances = train_ds[i, ...]
class_instance_address = class_instances[0].numpy().decode(
'utf-8')
class_address = os.path.split(class_instance_address)[0]
classes.add(class_address)
self.assertSetEqual(classes,
set(self.omniglot_database.train_folders.keys()))
def evaluate(self,
iterations,
iterations_to_load_from=None,
seed=-1,
num_tasks=1000):
# self.load_model(iterations=51, acc='0.78')
self.load_model()
self.test_dataset = self.get_test_dataset(seed=seed,
num_tasks=num_tasks)
task_number = 0
accs = list()
for (train_ds, val_ds), (train_labels,
val_labels) in self.test_dataset:
task_number += 1
if task_number % (num_tasks // 20) == 0:
print(f'{task_number} finished.')
train_labels = combine_first_two_axes(train_labels)
val_labels = combine_first_two_axes(val_labels)
train_labels = tf.math.argmax(train_labels, axis=1)
val_labels = tf.math.argmax(val_labels, axis=1)
train_ds = tf.squeeze(train_ds, axis=0)
train_ds = combine_first_two_axes(train_ds)
encodings = self.model.predict(train_ds)
from datetime import datetime
begin = datetime.now()
val_ds = tf.squeeze(val_ds, axis=0)
val_ds = combine_first_two_axes(val_ds)
val_encodings = self.model.predict(val_ds)
dists = self.euclidean_distance(val_encodings, encodings)
# predicted_labels = train_labels[tf.argmin(dists, axis=1)]
predicted_labels = train_labels.numpy()[tf.argmin(dists,
axis=1).numpy()]
end = datetime.now()
print(end - begin)
task_final_accuracy = np.mean(predicted_labels == val_labels)
accs.append(task_final_accuracy)
print(f'accuracy mean: {np.mean(accs)}')
print(f'accuracy std: {np.std(accs)}')
print(
f'final acc: {np.mean(accs)} +- {1.96 * np.std(accs) / np.sqrt(num_tasks)}'
)
return np.mean(accs)
def report_validation_loss_and_accuracy(self, epoch_count):
self.val_loss_metric.reset_states()
self.val_accuracy_metric.reset_states()
val_counter = 0
loss_func = self.get_losses_of_tasks_batch(method='val')
val_dataset = self.get_val_dataset()
for (train_ds, val_ds), (train_labels, val_labels) in val_dataset:
val_counter += 1
# TODO fix validation logging
if settings.DEBUG:
if val_counter % 5 == 0:
step = epoch_count * val_dataset.steps_per_epoch + val_counter
# pick the first task in meta batch
log_train_ds = combine_first_two_axes(train_ds[0, ...])
log_val_ds = combine_first_two_axes(val_ds[0, ...])
self.log_images(self.val_summary_writer, log_train_ds,
log_val_ds, step)
tasks_final_accuracy, tasks_final_losses = tf.map_fn(
loss_func,
elems=(
train_ds,
val_ds,
train_labels,
val_labels,
),
dtype=(tf.float32, tf.float32),
parallel_iterations=1)
final_loss = tf.reduce_mean(tasks_final_losses)
final_acc = tf.reduce_mean(tasks_final_accuracy)
self.val_loss_metric.update_state(final_loss)
self.val_accuracy_metric.update_state(final_acc)
self.log_metric(self.val_summary_writer,
'Loss',
self.val_loss_metric,
step=epoch_count)
self.log_metric(self.val_summary_writer,
'Accuracy',
self.val_accuracy_metric,
step=epoch_count)
print('Validation Loss: {}'.format(
self.val_loss_metric.result().numpy()))
print('Validation Accuracy: {}'.format(
self.val_accuracy_metric.result().numpy()))
def f(inputs):
train_ds, val_ds, train_labels, val_labels = inputs
train_ds = combine_first_two_axes(train_ds)
val_ds = combine_first_two_axes(val_ds)
self._initialize_eval_model()
for i in range(iterations):
self._train_model_for_eval(train_ds, train_labels)
train_ds = self.eval_model.get_features(
train_ds, training, apply_final_activation=False)
# train_ds = self.eval_model.predict(train_ds, training)
train_ds = tf.reshape(train_ds, (self.n, self.k_test, -1))
train_ds = tf.reduce_mean(train_ds, axis=1)
val_ds = self.eval_model.get_features(val_ds,
training,
apply_final_activation=False)
# val_ds = self.eval_model.predict(val_ds, training)
dists = euclidean_distance(train_ds, val_ds)
log_p_y = tf.transpose(tf.nn.log_softmax(-dists))
predicted_class_labels = tf.argmax(log_p_y, axis=-1)
real_val_labels = tf.argmax(val_labels, axis=-1)
val_acc = tf.reduce_mean(
tf.cast(tf.equal(predicted_class_labels, real_val_labels),
tf.float32))
val_loss = 0
if settings.DEBUG:
tf.print()
tf.print(val_loss)
tf.print(val_acc)
tf.print()
return val_acc, val_loss
def meta_train_loop(self, train_ds, val_ds, train_labels, val_labels):
tasks_final_losses = list()
tasks_final_accs = list()
weight_updates = dict()
for i in range(self.meta_batch_size):
task_train_set = combine_first_two_axes(train_ds[i, ...])
task_labels = train_labels[i, ...]
updated_model = self.inner_train_loop(task_train_set, task_labels)
# TODO evaluate it on task validation for training loss?
updated_model_logits = updated_model(task_train_set, training=True)
loss = self.outer_loss(task_labels, updated_model_logits)
tasks_final_losses.append(loss)
predicted_class_labels = self.predict_class_labels_from_logits(
updated_model_logits)
real_labels = self.convert_labels_to_real_labels(task_labels)
acc = tf.reduce_mean(
tf.cast(tf.equal(predicted_class_labels, real_labels),
tf.float32))
tasks_final_accs.append(acc)
for variable in self.model.trainable_variables:
references = self.extract_variable_reference_from_variable_name(
variable.name)
layer_names = references[:-1]
attr = references[-1]
model_layer = self.model
updated_model_layer = updated_model
for layer_name in layer_names:
model_layer = model_layer.get_layer(layer_name)
updated_model_layer = updated_model_layer.get_layer(
layer_name)
update_direction = updated_model_layer.__dict__[
attr] - model_layer.__dict__[attr]
weight_updates[variable.name] = weight_updates.get(
variable.name, tf.zeros_like(update_direction)
) + update_direction / self.meta_batch_size
gradients = []
for variable in self.model.trainable_variables:
if variable.name in weight_updates:
gradients.append(-1 * weight_updates[variable.name])
# references = self.extract_variable_reference_from_variable_name(variable.name)
# layer_names = references[:-1]
# attr = references[-1]
#
# model_layer = self.model
# for layer_name in layer_names:
# model_layer = model_layer.get_layer(layer_name)
#
# model_layer.__dict__[attr].assign(variable + weight_updates[variable.name])
# variable.assign(variable + self.meta_learning_rate * weight_updates[variable.name])
final_acc = tf.reduce_mean(tasks_final_accs)
final_loss = tf.reduce_mean(tasks_final_losses)
self.optimizer.apply_gradients(
zip(gradients, self.model.trainable_variables))
return final_acc, final_loss
def train(self, iterations=5):
self.train_summary_writer = tf.summary.create_file_writer(
self.train_log_dir)
self.val_summary_writer = tf.summary.create_file_writer(
self.val_log_dir)
train_dataset = self.get_train_dataset()
iteration_count = self.load_model()
epoch_count = iteration_count // tf.data.experimental.cardinality(
train_dataset)
pbar = tqdm(train_dataset)
train_accuracy_metric = tf.metrics.Mean()
train_accuracy_metric.reset_states()
train_loss_metric = tf.metrics.Mean()
train_loss_metric.reset_states()
should_continue = iteration_count < iterations
while should_continue:
for (train_ds, val_ds), (train_labels,
val_labels) in train_dataset:
train_acc, train_loss = self.meta_train_loop(
train_ds, val_ds, train_labels, val_labels)
train_accuracy_metric.update_state(train_acc)
train_loss_metric.update_state(train_loss)
iteration_count += 1
if (self.log_train_images_after_iteration != -1
and iteration_count %
self.log_train_images_after_iteration == 0):
self.log_images(self.train_summary_writer,
combine_first_two_axes(train_ds[0, ...]),
combine_first_two_axes(val_ds[0, ...]),
step=iteration_count)
self.log_histograms(step=iteration_count)
if iteration_count != 0 and iteration_count % self.save_after_iterations == 0:
self.save_model(iteration_count)
if iteration_count % self.report_validation_frequency == 0:
self.report_validation_loss_and_accuracy(iteration_count)
if iteration_count != 0:
print('Train Loss: {}'.format(
train_loss_metric.result().numpy()))
print('Train Accuracy: {}'.format(
train_accuracy_metric.result().numpy()))
with self.train_summary_writer.as_default():
tf.summary.scalar('Loss',
train_loss_metric.result(),
step=iteration_count)
tf.summary.scalar('Accuracy',
train_accuracy_metric.result(),
step=iteration_count)
train_accuracy_metric.reset_states()
train_loss_metric.reset_states()
pbar.set_description_str(
'Epoch{}, Iteration{}: Train Loss: {}, Train Accuracy: {}'.
format(epoch_count, iteration_count,
train_loss_metric.result().numpy(),
train_accuracy_metric.result().numpy()))
pbar.update(1)
if iteration_count >= iterations:
should_continue = False
break
epoch_count += 1
def evaluate(self,
iterations,
num_tasks,
k_test,
k_val_test,
inner_learning_rate,
seed=-1):
self.autoencoder.load_weights(
os.path.join(self.root, 'cross_domain',
f'model_{self.database.__class__}.h5'))
data_loader = BaseDataLoader(database=self.database,
val_database=self.database,
test_database=self.database,
n=5,
k_ml=1,
k_val_ml=1,
k_val=1,
k_val_val=15,
k_test=k_test,
k_val_test=k_val_test,
meta_batch_size=1,
num_tasks_val=100,
val_seed=42)
test_dataset = data_loader.get_test_dataset(num_tasks=num_tasks,
seed=seed)
print(self.database.__class__)
print(f'K_test: {k_test}')
print(f'K_test_val: {k_val_test}')
accs = list()
losses = list()
counter = 0
f = self.get_feature_parser()
optimizer = tf.keras.optimizers.SGD(learning_rate=inner_learning_rate)
for (train_ds, val_ds), (train_labels, val_labels) in test_dataset:
train_ds = combine_first_two_axes(tf.squeeze(train_ds, axis=0))
val_ds = combine_first_two_axes(tf.squeeze(val_ds, axis=0))
train_labels = combine_first_two_axes(train_labels)
val_labels = combine_first_two_axes(val_labels)
remainder_num = num_tasks // 20
if remainder_num == 0:
remainder_num = 1
if counter % remainder_num == 0:
print(f'{counter} / {num_tasks} are evaluated.')
counter += 1
train_feats = f(train_ds)
val_feats = f(val_ds)
domain_train_feats = self.encoder(train_feats)
rec_train_feats = self.decoder(
tf.keras.activations.relu(domain_train_feats))
print('here')
# domain_train_feats = train_feats
# tf.print(tf.reduce_max(domain_train_feats))
# tf.print(tf.reduce_min(domain_train_feats))
dense = tf.keras.layers.Dense(5, activation=None)
for inner_step in range(iterations):
with tf.GradientTape() as tape:
logits = dense(domain_train_feats)
loss = tf.reduce_mean(
tf.losses.categorical_crossentropy(train_labels,
logits,
from_logits=True))
gradients = tape.gradient(loss, dense.trainable_variables)
optimizer.apply_gradients(
zip(gradients, dense.trainable_variables))
domain_val_feats = self.encoder(val_feats)
# domain_val_feats = val_feats
val_logits = dense(domain_val_feats)
val_loss = tf.reduce_mean(
tf.losses.categorical_crossentropy(val_labels,
val_logits,
from_logits=True))
predicted_class_labels = tf.argmax(val_logits, axis=-1)
real_val_labels = tf.argmax(val_labels, axis=-1)
val_acc = tf.reduce_mean(
tf.cast(tf.equal(predicted_class_labels, real_val_labels),
tf.float32))
losses.append(val_loss)
accs.append(val_acc)
final_acc_mean = np.mean(accs)
final_acc_std = np.std(accs)
print(f'loss mean: {np.mean(losses)}')
print(f'loss std: {np.std(losses)}')
print(f'accuracy mean: {final_acc_mean}')
print(f'accuracy std: {final_acc_std}')
# Free the seed :D
if seed != -1:
np.random.seed(None)
confidence_interval = 1.96 * final_acc_std / np.sqrt(num_tasks)
print(f'final acc: {final_acc_mean} +- {confidence_interval}')
print(
f'final acc: {final_acc_mean * 100:0.2f} +- {confidence_interval * 100:0.2f}'
)
return np.mean(accs)
def train(self, iterations=5):
self.train_summary_writer = tf.summary.create_file_writer(
self.train_log_dir)
self.val_summary_writer = tf.summary.create_file_writer(
self.val_log_dir)
maml_train_dataset = self.ssml_maml.get_train_dataset()
maml_gan_train_dataset = self.get_train_dataset()
print("Not Loading any SSMLMAMLGAN model")
iteration_count = -1 # self.load_model()
N = 20
N_labeled = int(np.ceil(self.perc * N))
N_gen = int(N - N_labeled)
print(" #### ### ## # N_labeled:", N_labeled)
print(" #### ### ## # N_gen", N_gen)
print("Percentage of labeled data:", self.perc)
epoch_count = iteration_count // N
pbar = tqdm(maml_train_dataset)
train_accuracy_metric = tf.metrics.Mean()
train_accuracy_metric.reset_states()
train_loss_metric = tf.metrics.Mean()
train_loss_metric.reset_states()
should_continue = iteration_count < iterations
while should_continue:
for d, dataset in enumerate(
[maml_gan_train_dataset, maml_train_dataset]):
# print("DATASET:", ["generated data", "labeled data"][d])
N_dataset = [N_gen, N_labeled][d]
iteration_count_inner = 0
should_continue_inner = iteration_count_inner < N_dataset
while should_continue_inner:
for (train_ds, val_ds), (train_labels,
val_labels) in dataset:
train_acc, train_loss = self.meta_train_loop(
train_ds, val_ds, train_labels, val_labels)
train_accuracy_metric.update_state(train_acc)
train_loss_metric.update_state(train_loss)
iteration_count += 1
if (self.log_train_images_after_iteration != -1
and iteration_count %
self.log_train_images_after_iteration == 0):
self.log_images(
self.train_summary_writer,
combine_first_two_axes(train_ds[0, ...]),
combine_first_two_axes(val_ds[0, ...]),
step=iteration_count)
self.log_histograms(step=iteration_count)
if iteration_count != 0 and iteration_count % self.save_after_iterations == 0:
self.save_model(iteration_count)
if iteration_count % self.report_validation_frequency == 0:
self.report_validation_loss_and_accuracy(
iteration_count)
if iteration_count != 0:
print('Train Loss: {}'.format(
train_loss_metric.result().numpy()))
print('Train Accuracy: {}'.format(
train_accuracy_metric.result().numpy()))
with self.train_summary_writer.as_default():
tf.summary.scalar('Loss',
train_loss_metric.result(),
step=iteration_count)
tf.summary.scalar(
'Accuracy',
train_accuracy_metric.result(),
step=iteration_count)
train_accuracy_metric.reset_states()
train_loss_metric.reset_states()
pbar.set_description_str(
'Epoch{}, Iteration{}: Train Loss: {}, Train Accuracy: {}'
.format(epoch_count, iteration_count,
train_loss_metric.result().numpy(),
train_accuracy_metric.result().numpy()))
pbar.update(1)
iteration_count_inner += 1
if iteration_count_inner >= N_dataset:
should_continue_inner = False
break
if iteration_count >= iterations:
should_continue = False
break
epoch_count += 1
def evaluate_with_original_face_recognition(self,
iterations,
iterations_to_load_from=None,
num_tasks=1000,
seed=-1):
self.test_dataset = self.get_test_dataset(seed=seed,
num_tasks=num_tasks)
accs = list()
counter = 0
for (train_ds, val_ds), (train_labels,
val_labels) in self.test_dataset:
if counter % 50 == 0:
print(f'{counter} / {num_tasks} are evaluated.')
counter += 1
train_labels = combine_first_two_axes(train_labels)
val_labels = combine_first_two_axes(val_labels)
train_labels = tf.math.argmax(train_labels, axis=1)
val_labels = tf.math.argmax(val_labels, axis=1)
train_ds = tf.squeeze(train_ds, axis=0)
train_ds = combine_first_two_axes(train_ds)
encodings = []
labels = []
for image, label in zip(train_ds, train_labels):
image *= 255
image = image.numpy().astype(np.uint8)
encoding = face_recognition.face_encodings(image)
if len(encoding) > 0:
encoding = encoding[0]
encodings.append(encoding)
else:
print('bad')
encodings.append([0] * 128)
labels.append(label)
from datetime import datetime
begin = datetime.now()
val_ds = tf.squeeze(val_ds, axis=0)
val_ds = combine_first_two_axes(val_ds)
true_count = 0
all_count = 0
for image, label in zip(val_ds, val_labels):
image *= 255
image = image.numpy().astype(np.uint8)
encoding = face_recognition.face_encodings(image)
if len(encoding) > 0:
encoding = encoding[0]
face_distances = face_recognition.face_distance(
encodings, encoding)
best_match_index = np.argmin(face_distances)
if labels[best_match_index] == label:
true_count += 1
all_count += 1
end = datetime.now()
print(end - begin)
task_final_accuracy = true_count / all_count
accs.append(task_final_accuracy)
print(f'accuracy mean: {np.mean(accs)}')
print(f'accuracy std: {np.std(accs)}')
print(
f'final acc: {np.mean(accs)} +- {1.96 * np.std(accs) / np.sqrt(num_tasks)}'
)
return np.mean(accs)
def evaluate(self,
iterations,
num_tasks,
k_test,
k_val_test,
inner_learning_rate,
experiment_name,
seed=-1):
self.autoencoder.load_weights(os.path.join(
self.root, 'cross_domain',
f'model_{self.database.__class__}_{experiment_name}.h5'),
skip_mismatch=True,
by_name=True)
data_loader = BaseDataLoader(database=self.database,
val_database=self.database,
test_database=self.database,
n=5,
k_ml=1,
k_val_ml=1,
k_val=1,
k_val_val=15,
k_test=k_test,
k_val_test=k_val_test,
meta_batch_size=1,
num_tasks_val=100,
val_seed=42)
test_dataset = data_loader.get_test_dataset(num_tasks=num_tasks,
seed=seed)
print(self.database.__class__)
print(f'K_test: {k_test}')
print(f'K_test_val: {k_val_test}')
accs = list()
losses = list()
counter = 0
f = self.get_feature_parser()
optimizer = tf.keras.optimizers.SGD(learning_rate=inner_learning_rate)
# dense = MiniImagenetModel(num_classes=5)
# for layer in dense.layers:
# if isinstance(layer, tf.keras.layers.BatchNormalization):
# layer.momentum = 0.0
# dense = tf.keras.layers.Dense(5, activation=None)
test_encoder, test_decoder, test_autoencoder, = self.init_autoencoder()
for (train_ds, val_ds), (train_labels, val_labels) in test_dataset:
train_ds = combine_first_two_axes(tf.squeeze(train_ds, axis=0))
val_ds = combine_first_two_axes(tf.squeeze(val_ds, axis=0))
train_labels = combine_first_two_axes(train_labels)
val_labels = combine_first_two_axes(val_labels)
remainder_num = num_tasks // 20
if remainder_num == 0:
remainder_num = 1
if counter % remainder_num == 0:
print(f'{counter} / {num_tasks} are evaluated.')
counter += 1
train_feats = f(train_ds)
# train_feats = train_ds
val_feats = f(val_ds)
# val_feats = val_ds
# for inner_step in range(iterations):
# with tf.GradientTape() as tape:
# pass
test_autoencoder.set_weights(self.autoencoder.get_weights())
test_autoencoder.compile(
optimizer=tf.keras.optimizers.Adam(
learning_rate=inner_learning_rate),
loss=[
# None,
tf.keras.losses.CosineSimilarity(),
tf.keras.losses.CategoricalCrossentropy(from_logits=True)
])
test_autoencoder.fit(
train_feats, # 44.82 +- 0.56 wih noise 0.1
(train_feats, train_labels),
epochs=iterations,
verbose=0)
domain_train_feats = test_encoder.predict(train_feats)
# rec_train_feats = self.decoder(tf.keras.activations.relu(domain_train_feats))
# print('here')
# domain_train_feats = train_feats
domain_train_feats = tf.reshape(domain_train_feats,
(5, k_test, -1))
domain_train_feats = tf.reduce_mean(domain_train_feats, axis=1)
# tf.print(tf.reduce_max(domain_train_feats))
# tf.print(tf.reduce_min(domain_train_feats))
# dense(domain_train_feats)
# dense.set_weights(self.domain_models[0].layers[-1].get_weights())
# dense(domain_train_feats)
# dense.set_weights(self.domain_models[0].get_weights())
domain_val_feats = test_encoder.predict(val_feats)
# domain_val_feats = val_feats
# val_logits = dense(domain_val_feats)
dists = self.euclidean_distance(domain_train_feats,
domain_val_feats)
log_p_y = tf.transpose(tf.nn.log_softmax(-dists))
predicted_class_labels = tf.argmax(log_p_y, axis=-1)
# val_loss = tf.reduce_mean(
# tf.losses.categorical_crossentropy(val_labels, val_logits, from_logits=True)
# )
# predicted_class_labels = tf.argmax(val_logits, axis=-1)
real_val_labels = tf.argmax(val_labels, axis=-1)
val_acc = tf.reduce_mean(
tf.cast(tf.equal(predicted_class_labels, real_val_labels),
tf.float32))
losses.append(0)
accs.append(val_acc)
final_acc_mean = np.mean(accs)
final_acc_std = np.std(accs)
print(f'loss mean: {np.mean(losses)}')
print(f'loss std: {np.std(losses)}')
print(f'accuracy mean: {final_acc_mean}')
print(f'accuracy std: {final_acc_std}')
# Free the seed :D
if seed != -1:
np.random.seed(None)
confidence_interval = 1.96 * final_acc_std / np.sqrt(num_tasks)
print(f'final acc: {final_acc_mean} +- {confidence_interval}')
print(
f'final acc: {final_acc_mean * 100:0.2f} +- {confidence_interval * 100:0.2f}'
)
return np.mean(accs)
