def get_task(): meta_batch_vectors = list() for meta_batch in range(self.meta_batch_size): vectors = self.generate_all_vectors(latent_dim) vectors = tf.reshape(tf.stack(vectors, axis=0), (-1, latent_dim)) meta_batch_vectors.append(vectors) meta_batch_vectors = tf.stack(meta_batch_vectors) meta_batch_vectors = combine_first_two_axes(meta_batch_vectors) images = get_images_from_vectors(meta_batch_vectors) images = tf.image.resize(images, (84, 84)) images = tf.reshape( images, (self.meta_batch_size, self.n * (self.k + self.k_val_ml), *generated_image_shape)) train_ds = images[:, :self.n * self.k, ...] train_indices = [ i // self.k + i % self.k * self.n for i in range(self.n * self.k) ] train_ds = tf.gather(train_ds, train_indices, axis=1) train_ds = tf.reshape( train_ds, (self.meta_batch_size, self.n, self.k, *generated_image_shape)) val_ds = images[:, self.n * self.k:, ...] val_ds = combine_first_two_axes(val_ds) # random_num = tf.random.uniform(shape=(), minval=0, maxval=1) # if random_num < 4: # angles = tf.random.uniform( # shape=(self.meta_batch_size * self.n * self.k_val_ml, ), # minval=tf.constant(-np.pi), # maxval=tf.constant(np.pi) # ) # val_ds = tfa.image.rotate(val_ds, angles) # else: # val_ds = tf_image_translate( # val_ds, # tf.random.uniform((), -5, 5, dtype=tf.int32), # tf.random.uniform((), -5, 5, dtype=tf.int32) # ) val_ds = tf.reshape(val_ds, (self.meta_batch_size, self.n * self.k_val_ml, *generated_image_shape)) val_indices = [ i // self.k_val_ml + i % self.k_val_ml * self.n for i in range(self.n * self.k_val_ml) ] val_ds = tf.gather(val_ds, val_indices, axis=1) val_ds = tf.reshape(val_ds, (self.meta_batch_size, self.n, self.k_val_ml, *generated_image_shape)) yield (train_ds, val_ds), (train_labels, val_labels)
def get_task_train_and_val_ds(self, task, labels): train_ds, val_ds = tf.split(task, num_or_size_splits=2) train_labels, val_labels = tf.split(labels, num_or_size_splits=2) train_ds = combine_first_two_axes(tf.squeeze(train_ds, axis=0)) val_ds = combine_first_two_axes(tf.squeeze(val_ds, axis=0)) train_labels = combine_first_two_axes(tf.squeeze(train_labels, axis=0)) val_labels = combine_first_two_axes(tf.squeeze(val_labels, axis=0)) return train_ds, val_ds, train_labels, val_labels
def f(inputs): train_ds, train_dom, val_ds, val_dom, train_labels, val_labels = inputs train_ds = combine_first_two_axes(train_ds) train_dom = combine_first_two_axes(train_dom) val_ds = combine_first_two_axes(val_ds) val_dom = combine_first_two_axes(val_dom) self._initialize_eval_model() for i in range(iterations): self._train_model_for_eval(train_ds, train_dom, train_labels) val_acc, val_loss = self._evaluate_model_for_eval(val_ds, val_dom, val_labels, training) return val_acc, val_loss
def evaluate(self, iterations, iterations_to_load_from=None, seed=-1): # self.load_model(iterations=51, acc='0.78') # self.load_model() # model = load_model('facenet_keras.h5') model = load_model( '/home/siavash/PycharmProjects/facenet/model/facenet_keras.h5') print(model.inputs) print(model.outputs) model.trainable = False self.model = model self.model.summary() # dense = tf.keras.layers.Dense(self.n, activation='softmax')(model.output) # self.model = tf.keras.models.Model(inputs=model.inputs, outputs=dense) self.test_dataset = self.get_test_dataset(seed) task_number = 0 accs = list() for (train_ds, val_ds), (train_labels, val_labels) in self.test_dataset: task_number += 1 if task_number % (self.number_of_tasks_test // 20) == 0: print(f'{task_number} finished.') train_labels = combine_first_two_axes(train_labels) val_labels = combine_first_two_axes(val_labels) train_labels = tf.math.argmax(train_labels, axis=1) val_labels = tf.math.argmax(val_labels, axis=1) train_ds = tf.squeeze(train_ds, axis=0) train_ds = combine_first_two_axes(train_ds) encodings = self.model.predict(train_ds) from datetime import datetime begin = datetime.now() val_ds = tf.squeeze(val_ds, axis=0) val_ds = combine_first_two_axes(val_ds) val_encodings = self.model.predict(val_ds) dists = self.euclidean_distance(val_encodings, encodings) # predicted_labels = train_labels[tf.argmin(dists, axis=1)] predicted_labels = train_labels.numpy()[tf.argmin(dists, axis=1).numpy()] end = datetime.now() print(end - begin) task_final_accuracy = np.mean(predicted_labels == val_labels) accs.append(task_final_accuracy) print(f'accuracy mean: {np.mean(accs)}') print(f'accuracy std: {np.std(accs)}') print( f'final acc: {np.mean(accs)} +- {1.96 * np.std(accs) / np.sqrt(self.number_of_tasks_test)}' ) return np.mean(accs)
def get_task(): meta_batch_vectors = list() for meta_batch in range(self.meta_batch_size): vectors = self.generate_all_vectors() vectors = tf.reshape(tf.stack(vectors, axis=0), (-1, self.latent_dim)) meta_batch_vectors.append(vectors) meta_batch_vectors = tf.stack(meta_batch_vectors) meta_batch_vectors = combine_first_two_axes(meta_batch_vectors) images = self.get_images_from_vectors(meta_batch_vectors) images = tf.image.resize(images, self.generated_image_shape[:2]) images = tf.reshape( images, (self.meta_batch_size, self.n * (self.k + self.k_val_ml), *self.generated_image_shape)) train_ds = images[:, :self.n * self.k, ...] train_ds = tf.gather(train_ds, train_indices, axis=1) train_ds = tf.reshape(train_ds, (self.meta_batch_size, self.n, self.k, *self.generated_image_shape)) val_ds = images[:, self.n * self.k:, ...] val_ds = combine_first_two_axes(val_ds) # ================ val_imgs = list() for i in range(val_ds.shape[0]): val_image = val_ds[i, ...] tx = tf.random.uniform((), -5, 5, dtype=tf.int32) ty = tf.random.uniform((), -5, 5, dtype=tf.int32) transforms = [1, 0, -tx, 0, 1, -ty, 0, 0] val_image = tfa.image.transform(val_image, transforms, 'NEAREST') val_imgs.append(val_image) val_ds = tf.stack(val_imgs, axis=0) # ================ val_ds = tf.reshape(val_ds, (self.meta_batch_size, self.n * self.k_val_ml, *self.generated_image_shape)) val_ds = tf.gather(val_ds, val_indices, axis=1) val_ds = tf.reshape(val_ds, (self.meta_batch_size, self.n, self.k_val_ml, *self.generated_image_shape)) yield (train_ds, val_ds), (train_labels, val_labels)
def f(inputs): train_ds, val_ds, train_labels, val_labels = inputs train_ds = combine_first_two_axes(train_ds) val_ds = combine_first_two_axes(val_ds) self._initialize_eval_model() for i in range(iterations): self._train_model_for_eval(train_ds, train_labels) val_acc, val_loss = self._evaluate_model_for_eval(val_ds, val_labels, training) if settings.DEBUG: tf.print() tf.print(val_loss) tf.print(val_acc) tf.print() return val_acc, val_loss
def get_task(resolution=(25, 25, 3), tf_seed=3, n=5): # return (train_ds, val_ds), (train_labels, val_labels) latent_dim = 512 # 100 for omniglot generated_image_shape = resolution # (28, 28, 1) for omniglot vectors, noise_vectors = generate_all_vectors_by_interpolation(latent_dim, tf_seed, n=n) vectors = tf.reshape(tf.stack(vectors, axis=0), (-1, latent_dim)) images = gan_generator(vectors)['default'] images = tf.image.resize(images, generated_image_shape[:2]) train_ds = images[:n * k] train_indices = [i // k + i % k * n for i in range(n * k)] train_ds = tf.gather(train_ds, train_indices, axis=0) train_ds = tf.reshape(train_ds, (n, k, *generated_image_shape)) val_ds = images[n * k:] val_indices = [i // k_val_ml + i % k_val_ml * n for i in range(n * k_val_ml)] val_ds = tf.gather(val_ds, val_indices, axis=0) val_ds = tf.reshape(val_ds, (n, k_val_ml, *generated_image_shape)) val_ds = combine_first_two_axes(val_ds) val_ds = tf.reshape(val_ds, (n, k_val_ml, *generated_image_shape)) noise_images = gan_generator(noise_vectors)['default'] noise_images = tf.image.resize(noise_images, generated_image_shape[:2]) train_labels = np.repeat(np.arange(n), k) val_labels = np.repeat(np.arange(n), k_val_ml) train_labels = tf.one_hot(train_labels, depth=n) val_labels = tf.one_hot(val_labels, depth=n) yield (train_ds, val_ds), (train_labels, val_labels), vectors, noise_vectors, noise_images
def f(inputs): train_ds, val_ds, train_labels, val_labels = inputs train_ds = combine_first_two_axes(train_ds) val_ds = combine_first_two_axes(val_ds) updated_model, inner_losses = self.inner_train_loop(train_ds, train_labels) # TODO test what happens when training=False updated_model_logits = updated_model(val_ds, training=True) val_loss = self.outer_loss(val_labels, updated_model_logits, inner_losses) predicted_class_labels = self.predict_class_labels_from_logits(updated_model_logits) real_labels = self.convert_labels_to_real_labels(val_labels) val_acc = tf.reduce_mean(tf.cast(tf.equal(predicted_class_labels, real_labels), tf.float32)) return val_acc, val_loss
def test_covering_all_classes_in_one_epoch(self, mocked_parse_function): # Make a new database so that the number of classes are dividable by the number of meta batches * n. mocked_parse_function.return_value = self.parse_function ds = self.omniglot_database.get_supervised_meta_learning_dataset( self.omniglot_database.train_folders, n=5, k=5, k_validation=5, meta_batch_size=4, dtype=tf.string) # Check for covering all classes and no duplicates classes = set() for (x, x_val), (y, y_val) in ds: train_ds = combine_first_two_axes(x) for i in range(train_ds.shape[0]): class_instances = train_ds[i, ...] class_instance_address = class_instances[0].numpy().decode( 'utf-8') class_address = os.path.split(class_instance_address)[0] self.assertNotIn(class_address, classes) classes.add(class_address) self.assertSetEqual(classes, set(self.omniglot_database.train_folders.keys()))
def test_covering_all_classes_in_subsequent_epochs(self, mocked_parse_function): # This test might not pass because of the random selection of classes at the beginning of the class, but the # chances are low. Specially if we increase the number of epochs, the chance of not covering classes will # decrease. Make a new database so that the number of classes are dividable by the number of meta batches * n. # This test should always fail with num_epochs = 1 num_epochs = 3 mocked_parse_function.return_value = self.parse_function ds = self.omniglot_database.get_supervised_meta_learning_dataset( self.omniglot_database.train_folders, n=7, k=5, k_validation=5, meta_batch_size=4, seed=42, dtype=tf.string) # Check for covering all classes classes = set() for epoch in range(num_epochs): for (x, x_val), (y, y_val) in ds: train_ds = combine_first_two_axes(x) for i in range(train_ds.shape[0]): class_instances = train_ds[i, ...] class_instance_address = class_instances[0].numpy().decode( 'utf-8') class_address = os.path.split(class_instance_address)[0] classes.add(class_address) self.assertSetEqual(classes, set(self.omniglot_database.train_folders.keys()))
def evaluate(self, iterations, iterations_to_load_from=None, seed=-1, num_tasks=1000): # self.load_model(iterations=51, acc='0.78') self.load_model() self.test_dataset = self.get_test_dataset(seed=seed, num_tasks=num_tasks) task_number = 0 accs = list() for (train_ds, val_ds), (train_labels, val_labels) in self.test_dataset: task_number += 1 if task_number % (num_tasks // 20) == 0: print(f'{task_number} finished.') train_labels = combine_first_two_axes(train_labels) val_labels = combine_first_two_axes(val_labels) train_labels = tf.math.argmax(train_labels, axis=1) val_labels = tf.math.argmax(val_labels, axis=1) train_ds = tf.squeeze(train_ds, axis=0) train_ds = combine_first_two_axes(train_ds) encodings = self.model.predict(train_ds) from datetime import datetime begin = datetime.now() val_ds = tf.squeeze(val_ds, axis=0) val_ds = combine_first_two_axes(val_ds) val_encodings = self.model.predict(val_ds) dists = self.euclidean_distance(val_encodings, encodings) # predicted_labels = train_labels[tf.argmin(dists, axis=1)] predicted_labels = train_labels.numpy()[tf.argmin(dists, axis=1).numpy()] end = datetime.now() print(end - begin) task_final_accuracy = np.mean(predicted_labels == val_labels) accs.append(task_final_accuracy) print(f'accuracy mean: {np.mean(accs)}') print(f'accuracy std: {np.std(accs)}') print( f'final acc: {np.mean(accs)} +- {1.96 * np.std(accs) / np.sqrt(num_tasks)}' ) return np.mean(accs)
def report_validation_loss_and_accuracy(self, epoch_count): self.val_loss_metric.reset_states() self.val_accuracy_metric.reset_states() val_counter = 0 loss_func = self.get_losses_of_tasks_batch(method='val') val_dataset = self.get_val_dataset() for (train_ds, val_ds), (train_labels, val_labels) in val_dataset: val_counter += 1 # TODO fix validation logging if settings.DEBUG: if val_counter % 5 == 0: step = epoch_count * val_dataset.steps_per_epoch + val_counter # pick the first task in meta batch log_train_ds = combine_first_two_axes(train_ds[0, ...]) log_val_ds = combine_first_two_axes(val_ds[0, ...]) self.log_images(self.val_summary_writer, log_train_ds, log_val_ds, step) tasks_final_accuracy, tasks_final_losses = tf.map_fn( loss_func, elems=( train_ds, val_ds, train_labels, val_labels, ), dtype=(tf.float32, tf.float32), parallel_iterations=1) final_loss = tf.reduce_mean(tasks_final_losses) final_acc = tf.reduce_mean(tasks_final_accuracy) self.val_loss_metric.update_state(final_loss) self.val_accuracy_metric.update_state(final_acc) self.log_metric(self.val_summary_writer, 'Loss', self.val_loss_metric, step=epoch_count) self.log_metric(self.val_summary_writer, 'Accuracy', self.val_accuracy_metric, step=epoch_count) print('Validation Loss: {}'.format( self.val_loss_metric.result().numpy())) print('Validation Accuracy: {}'.format( self.val_accuracy_metric.result().numpy()))
def f(inputs): train_ds, val_ds, train_labels, val_labels = inputs train_ds = combine_first_two_axes(train_ds) val_ds = combine_first_two_axes(val_ds) self._initialize_eval_model() for i in range(iterations): self._train_model_for_eval(train_ds, train_labels) train_ds = self.eval_model.get_features( train_ds, training, apply_final_activation=False) # train_ds = self.eval_model.predict(train_ds, training) train_ds = tf.reshape(train_ds, (self.n, self.k_test, -1)) train_ds = tf.reduce_mean(train_ds, axis=1) val_ds = self.eval_model.get_features(val_ds, training, apply_final_activation=False) # val_ds = self.eval_model.predict(val_ds, training) dists = euclidean_distance(train_ds, val_ds) log_p_y = tf.transpose(tf.nn.log_softmax(-dists)) predicted_class_labels = tf.argmax(log_p_y, axis=-1) real_val_labels = tf.argmax(val_labels, axis=-1) val_acc = tf.reduce_mean( tf.cast(tf.equal(predicted_class_labels, real_val_labels), tf.float32)) val_loss = 0 if settings.DEBUG: tf.print() tf.print(val_loss) tf.print(val_acc) tf.print() return val_acc, val_loss
def meta_train_loop(self, train_ds, val_ds, train_labels, val_labels): tasks_final_losses = list() tasks_final_accs = list() weight_updates = dict() for i in range(self.meta_batch_size): task_train_set = combine_first_two_axes(train_ds[i, ...]) task_labels = train_labels[i, ...] updated_model = self.inner_train_loop(task_train_set, task_labels) # TODO evaluate it on task validation for training loss? updated_model_logits = updated_model(task_train_set, training=True) loss = self.outer_loss(task_labels, updated_model_logits) tasks_final_losses.append(loss) predicted_class_labels = self.predict_class_labels_from_logits( updated_model_logits) real_labels = self.convert_labels_to_real_labels(task_labels) acc = tf.reduce_mean( tf.cast(tf.equal(predicted_class_labels, real_labels), tf.float32)) tasks_final_accs.append(acc) for variable in self.model.trainable_variables: references = self.extract_variable_reference_from_variable_name( variable.name) layer_names = references[:-1] attr = references[-1] model_layer = self.model updated_model_layer = updated_model for layer_name in layer_names: model_layer = model_layer.get_layer(layer_name) updated_model_layer = updated_model_layer.get_layer( layer_name) update_direction = updated_model_layer.__dict__[ attr] - model_layer.__dict__[attr] weight_updates[variable.name] = weight_updates.get( variable.name, tf.zeros_like(update_direction) ) + update_direction / self.meta_batch_size gradients = [] for variable in self.model.trainable_variables: if variable.name in weight_updates: gradients.append(-1 * weight_updates[variable.name]) # references = self.extract_variable_reference_from_variable_name(variable.name) # layer_names = references[:-1] # attr = references[-1] # # model_layer = self.model # for layer_name in layer_names: # model_layer = model_layer.get_layer(layer_name) # # model_layer.__dict__[attr].assign(variable + weight_updates[variable.name]) # variable.assign(variable + self.meta_learning_rate * weight_updates[variable.name]) final_acc = tf.reduce_mean(tasks_final_accs) final_loss = tf.reduce_mean(tasks_final_losses) self.optimizer.apply_gradients( zip(gradients, self.model.trainable_variables)) return final_acc, final_loss
def train(self, iterations=5): self.train_summary_writer = tf.summary.create_file_writer( self.train_log_dir) self.val_summary_writer = tf.summary.create_file_writer( self.val_log_dir) train_dataset = self.get_train_dataset() iteration_count = self.load_model() epoch_count = iteration_count // tf.data.experimental.cardinality( train_dataset) pbar = tqdm(train_dataset) train_accuracy_metric = tf.metrics.Mean() train_accuracy_metric.reset_states() train_loss_metric = tf.metrics.Mean() train_loss_metric.reset_states() should_continue = iteration_count < iterations while should_continue: for (train_ds, val_ds), (train_labels, val_labels) in train_dataset: train_acc, train_loss = self.meta_train_loop( train_ds, val_ds, train_labels, val_labels) train_accuracy_metric.update_state(train_acc) train_loss_metric.update_state(train_loss) iteration_count += 1 if (self.log_train_images_after_iteration != -1 and iteration_count % self.log_train_images_after_iteration == 0): self.log_images(self.train_summary_writer, combine_first_two_axes(train_ds[0, ...]), combine_first_two_axes(val_ds[0, ...]), step=iteration_count) self.log_histograms(step=iteration_count) if iteration_count != 0 and iteration_count % self.save_after_iterations == 0: self.save_model(iteration_count) if iteration_count % self.report_validation_frequency == 0: self.report_validation_loss_and_accuracy(iteration_count) if iteration_count != 0: print('Train Loss: {}'.format( train_loss_metric.result().numpy())) print('Train Accuracy: {}'.format( train_accuracy_metric.result().numpy())) with self.train_summary_writer.as_default(): tf.summary.scalar('Loss', train_loss_metric.result(), step=iteration_count) tf.summary.scalar('Accuracy', train_accuracy_metric.result(), step=iteration_count) train_accuracy_metric.reset_states() train_loss_metric.reset_states() pbar.set_description_str( 'Epoch{}, Iteration{}: Train Loss: {}, Train Accuracy: {}'. format(epoch_count, iteration_count, train_loss_metric.result().numpy(), train_accuracy_metric.result().numpy())) pbar.update(1) if iteration_count >= iterations: should_continue = False break epoch_count += 1
def evaluate(self, iterations, num_tasks, k_test, k_val_test, inner_learning_rate, seed=-1): self.autoencoder.load_weights( os.path.join(self.root, 'cross_domain', f'model_{self.database.__class__}.h5')) data_loader = BaseDataLoader(database=self.database, val_database=self.database, test_database=self.database, n=5, k_ml=1, k_val_ml=1, k_val=1, k_val_val=15, k_test=k_test, k_val_test=k_val_test, meta_batch_size=1, num_tasks_val=100, val_seed=42) test_dataset = data_loader.get_test_dataset(num_tasks=num_tasks, seed=seed) print(self.database.__class__) print(f'K_test: {k_test}') print(f'K_test_val: {k_val_test}') accs = list() losses = list() counter = 0 f = self.get_feature_parser() optimizer = tf.keras.optimizers.SGD(learning_rate=inner_learning_rate) for (train_ds, val_ds), (train_labels, val_labels) in test_dataset: train_ds = combine_first_two_axes(tf.squeeze(train_ds, axis=0)) val_ds = combine_first_two_axes(tf.squeeze(val_ds, axis=0)) train_labels = combine_first_two_axes(train_labels) val_labels = combine_first_two_axes(val_labels) remainder_num = num_tasks // 20 if remainder_num == 0: remainder_num = 1 if counter % remainder_num == 0: print(f'{counter} / {num_tasks} are evaluated.') counter += 1 train_feats = f(train_ds) val_feats = f(val_ds) domain_train_feats = self.encoder(train_feats) rec_train_feats = self.decoder( tf.keras.activations.relu(domain_train_feats)) print('here') # domain_train_feats = train_feats # tf.print(tf.reduce_max(domain_train_feats)) # tf.print(tf.reduce_min(domain_train_feats)) dense = tf.keras.layers.Dense(5, activation=None) for inner_step in range(iterations): with tf.GradientTape() as tape: logits = dense(domain_train_feats) loss = tf.reduce_mean( tf.losses.categorical_crossentropy(train_labels, logits, from_logits=True)) gradients = tape.gradient(loss, dense.trainable_variables) optimizer.apply_gradients( zip(gradients, dense.trainable_variables)) domain_val_feats = self.encoder(val_feats) # domain_val_feats = val_feats val_logits = dense(domain_val_feats) val_loss = tf.reduce_mean( tf.losses.categorical_crossentropy(val_labels, val_logits, from_logits=True)) predicted_class_labels = tf.argmax(val_logits, axis=-1) real_val_labels = tf.argmax(val_labels, axis=-1) val_acc = tf.reduce_mean( tf.cast(tf.equal(predicted_class_labels, real_val_labels), tf.float32)) losses.append(val_loss) accs.append(val_acc) final_acc_mean = np.mean(accs) final_acc_std = np.std(accs) print(f'loss mean: {np.mean(losses)}') print(f'loss std: {np.std(losses)}') print(f'accuracy mean: {final_acc_mean}') print(f'accuracy std: {final_acc_std}') # Free the seed :D if seed != -1: np.random.seed(None) confidence_interval = 1.96 * final_acc_std / np.sqrt(num_tasks) print(f'final acc: {final_acc_mean} +- {confidence_interval}') print( f'final acc: {final_acc_mean * 100:0.2f} +- {confidence_interval * 100:0.2f}' ) return np.mean(accs)
def train(self, iterations=5): self.train_summary_writer = tf.summary.create_file_writer( self.train_log_dir) self.val_summary_writer = tf.summary.create_file_writer( self.val_log_dir) maml_train_dataset = self.ssml_maml.get_train_dataset() maml_gan_train_dataset = self.get_train_dataset() print("Not Loading any SSMLMAMLGAN model") iteration_count = -1 # self.load_model() N = 20 N_labeled = int(np.ceil(self.perc * N)) N_gen = int(N - N_labeled) print(" #### ### ## # N_labeled:", N_labeled) print(" #### ### ## # N_gen", N_gen) print("Percentage of labeled data:", self.perc) epoch_count = iteration_count // N pbar = tqdm(maml_train_dataset) train_accuracy_metric = tf.metrics.Mean() train_accuracy_metric.reset_states() train_loss_metric = tf.metrics.Mean() train_loss_metric.reset_states() should_continue = iteration_count < iterations while should_continue: for d, dataset in enumerate( [maml_gan_train_dataset, maml_train_dataset]): # print("DATASET:", ["generated data", "labeled data"][d]) N_dataset = [N_gen, N_labeled][d] iteration_count_inner = 0 should_continue_inner = iteration_count_inner < N_dataset while should_continue_inner: for (train_ds, val_ds), (train_labels, val_labels) in dataset: train_acc, train_loss = self.meta_train_loop( train_ds, val_ds, train_labels, val_labels) train_accuracy_metric.update_state(train_acc) train_loss_metric.update_state(train_loss) iteration_count += 1 if (self.log_train_images_after_iteration != -1 and iteration_count % self.log_train_images_after_iteration == 0): self.log_images( self.train_summary_writer, combine_first_two_axes(train_ds[0, ...]), combine_first_two_axes(val_ds[0, ...]), step=iteration_count) self.log_histograms(step=iteration_count) if iteration_count != 0 and iteration_count % self.save_after_iterations == 0: self.save_model(iteration_count) if iteration_count % self.report_validation_frequency == 0: self.report_validation_loss_and_accuracy( iteration_count) if iteration_count != 0: print('Train Loss: {}'.format( train_loss_metric.result().numpy())) print('Train Accuracy: {}'.format( train_accuracy_metric.result().numpy())) with self.train_summary_writer.as_default(): tf.summary.scalar('Loss', train_loss_metric.result(), step=iteration_count) tf.summary.scalar( 'Accuracy', train_accuracy_metric.result(), step=iteration_count) train_accuracy_metric.reset_states() train_loss_metric.reset_states() pbar.set_description_str( 'Epoch{}, Iteration{}: Train Loss: {}, Train Accuracy: {}' .format(epoch_count, iteration_count, train_loss_metric.result().numpy(), train_accuracy_metric.result().numpy())) pbar.update(1) iteration_count_inner += 1 if iteration_count_inner >= N_dataset: should_continue_inner = False break if iteration_count >= iterations: should_continue = False break epoch_count += 1
def evaluate_with_original_face_recognition(self, iterations, iterations_to_load_from=None, num_tasks=1000, seed=-1): self.test_dataset = self.get_test_dataset(seed=seed, num_tasks=num_tasks) accs = list() counter = 0 for (train_ds, val_ds), (train_labels, val_labels) in self.test_dataset: if counter % 50 == 0: print(f'{counter} / {num_tasks} are evaluated.') counter += 1 train_labels = combine_first_two_axes(train_labels) val_labels = combine_first_two_axes(val_labels) train_labels = tf.math.argmax(train_labels, axis=1) val_labels = tf.math.argmax(val_labels, axis=1) train_ds = tf.squeeze(train_ds, axis=0) train_ds = combine_first_two_axes(train_ds) encodings = [] labels = [] for image, label in zip(train_ds, train_labels): image *= 255 image = image.numpy().astype(np.uint8) encoding = face_recognition.face_encodings(image) if len(encoding) > 0: encoding = encoding[0] encodings.append(encoding) else: print('bad') encodings.append([0] * 128) labels.append(label) from datetime import datetime begin = datetime.now() val_ds = tf.squeeze(val_ds, axis=0) val_ds = combine_first_two_axes(val_ds) true_count = 0 all_count = 0 for image, label in zip(val_ds, val_labels): image *= 255 image = image.numpy().astype(np.uint8) encoding = face_recognition.face_encodings(image) if len(encoding) > 0: encoding = encoding[0] face_distances = face_recognition.face_distance( encodings, encoding) best_match_index = np.argmin(face_distances) if labels[best_match_index] == label: true_count += 1 all_count += 1 end = datetime.now() print(end - begin) task_final_accuracy = true_count / all_count accs.append(task_final_accuracy) print(f'accuracy mean: {np.mean(accs)}') print(f'accuracy std: {np.std(accs)}') print( f'final acc: {np.mean(accs)} +- {1.96 * np.std(accs) / np.sqrt(num_tasks)}' ) return np.mean(accs)
def evaluate(self, iterations, num_tasks, k_test, k_val_test, inner_learning_rate, experiment_name, seed=-1): self.autoencoder.load_weights(os.path.join( self.root, 'cross_domain', f'model_{self.database.__class__}_{experiment_name}.h5'), skip_mismatch=True, by_name=True) data_loader = BaseDataLoader(database=self.database, val_database=self.database, test_database=self.database, n=5, k_ml=1, k_val_ml=1, k_val=1, k_val_val=15, k_test=k_test, k_val_test=k_val_test, meta_batch_size=1, num_tasks_val=100, val_seed=42) test_dataset = data_loader.get_test_dataset(num_tasks=num_tasks, seed=seed) print(self.database.__class__) print(f'K_test: {k_test}') print(f'K_test_val: {k_val_test}') accs = list() losses = list() counter = 0 f = self.get_feature_parser() optimizer = tf.keras.optimizers.SGD(learning_rate=inner_learning_rate) # dense = MiniImagenetModel(num_classes=5) # for layer in dense.layers: # if isinstance(layer, tf.keras.layers.BatchNormalization): # layer.momentum = 0.0 # dense = tf.keras.layers.Dense(5, activation=None) test_encoder, test_decoder, test_autoencoder, = self.init_autoencoder() for (train_ds, val_ds), (train_labels, val_labels) in test_dataset: train_ds = combine_first_two_axes(tf.squeeze(train_ds, axis=0)) val_ds = combine_first_two_axes(tf.squeeze(val_ds, axis=0)) train_labels = combine_first_two_axes(train_labels) val_labels = combine_first_two_axes(val_labels) remainder_num = num_tasks // 20 if remainder_num == 0: remainder_num = 1 if counter % remainder_num == 0: print(f'{counter} / {num_tasks} are evaluated.') counter += 1 train_feats = f(train_ds) # train_feats = train_ds val_feats = f(val_ds) # val_feats = val_ds # for inner_step in range(iterations): # with tf.GradientTape() as tape: # pass test_autoencoder.set_weights(self.autoencoder.get_weights()) test_autoencoder.compile( optimizer=tf.keras.optimizers.Adam( learning_rate=inner_learning_rate), loss=[ # None, tf.keras.losses.CosineSimilarity(), tf.keras.losses.CategoricalCrossentropy(from_logits=True) ]) test_autoencoder.fit( train_feats, # 44.82 +- 0.56 wih noise 0.1 (train_feats, train_labels), epochs=iterations, verbose=0) domain_train_feats = test_encoder.predict(train_feats) # rec_train_feats = self.decoder(tf.keras.activations.relu(domain_train_feats)) # print('here') # domain_train_feats = train_feats domain_train_feats = tf.reshape(domain_train_feats, (5, k_test, -1)) domain_train_feats = tf.reduce_mean(domain_train_feats, axis=1) # tf.print(tf.reduce_max(domain_train_feats)) # tf.print(tf.reduce_min(domain_train_feats)) # dense(domain_train_feats) # dense.set_weights(self.domain_models[0].layers[-1].get_weights()) # dense(domain_train_feats) # dense.set_weights(self.domain_models[0].get_weights()) domain_val_feats = test_encoder.predict(val_feats) # domain_val_feats = val_feats # val_logits = dense(domain_val_feats) dists = self.euclidean_distance(domain_train_feats, domain_val_feats) log_p_y = tf.transpose(tf.nn.log_softmax(-dists)) predicted_class_labels = tf.argmax(log_p_y, axis=-1) # val_loss = tf.reduce_mean( # tf.losses.categorical_crossentropy(val_labels, val_logits, from_logits=True) # ) # predicted_class_labels = tf.argmax(val_logits, axis=-1) real_val_labels = tf.argmax(val_labels, axis=-1) val_acc = tf.reduce_mean( tf.cast(tf.equal(predicted_class_labels, real_val_labels), tf.float32)) losses.append(0) accs.append(val_acc) final_acc_mean = np.mean(accs) final_acc_std = np.std(accs) print(f'loss mean: {np.mean(losses)}') print(f'loss std: {np.std(losses)}') print(f'accuracy mean: {final_acc_mean}') print(f'accuracy std: {final_acc_std}') # Free the seed :D if seed != -1: np.random.seed(None) confidence_interval = 1.96 * final_acc_std / np.sqrt(num_tasks) print(f'final acc: {final_acc_mean} +- {confidence_interval}') print( f'final acc: {final_acc_mean * 100:0.2f} +- {confidence_interval * 100:0.2f}' ) return np.mean(accs)