def pretrain(sort_samples=True, model_name="mrcl"): print(f"GPU is available: {tf.test.is_gpu_available()}") background_data, _ = load_omniglot(verbose=1) background_training_data, _, _ = get_background_data_by_classes(background_data, sort=sort_samples) s_learn, s_remember = partition_into_disjoint(background_training_data) rln, tln = mrcl_omniglot() t = range(15000) current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") train_log_dir = 'logs/classification/pretraining/omniglot/' + model_name + '/gradient_tape/' + current_time + '/train' train_summary_writer = tf.summary.create_file_writer(train_log_dir) tln_initial = tf.keras.models.clone_model(tln) for epoch, v in enumerate(t): x_rand, y_rand = sample_random_10_classes(s_remember, background_training_data) x_traj, y_traj = sample_trajectory(s_learn, background_training_data) loss = pretrain_classification_mrcl(x_traj, y_traj, x_rand, y_rand, rln, tln, tln_initial, classification_parameters) # Check metrics rep = rln(x_rand) rep = np.array(rep) counts = np.isclose(rep, 0).sum(axis=1) / rep.shape[1] sparsity = np.mean(counts) with train_summary_writer.as_default(): tf.summary.scalar('Sparsity', sparsity, step=epoch) tf.summary.scalar('Training loss', loss, step=epoch) if (epoch+1) % 1000 == 0: print("Epoch:", epoch, "Sparsity:", sparsity, "Training loss:", loss.numpy()) save_models(tln, f"tln_pretraining_{model_name}_{epoch}_omniglot") save_models(rln, f"rln_pretraining_{model_name}_{epoch}_omniglot")
def build_omniglot_model(self, n_layers_rln=6, n_layers_tln=2, filters=256, hidden_units_per_layer=300, seed=None): self.model_rln, self.model_tln = mrcl_omniglot(n_layers_rln, n_layers_tln, filters, hidden_units_per_layer, seed) self.compute_loss_training = tf.function(self._compute_loss) self.compute_loss_no_training = tf.function( self._compute_loss_no_regularization)
def evaluate(model_name, model_type="mrcl"): _, evaluation_data = load_omniglot(verbose=1) evaluation_training_data, evaluation_test_data = get_eval_data_by_classes( evaluation_data) save_dir = "results/omniglot/" + model_type try: os.stat(save_dir) except IOError: os.mkdir(save_dir) points = [10, 50, 75, 100, 150, 200] for point in points: lrs = [ 0.3, 0.1, 0.03, 0.01, 0.003, 0.001, 0.0003, 0.0001, 0.00003, 0.00001 ] test_accuracy_results = [] train_accuracy_results = [] for lr in lrs: tf.keras.backend.clear_session() _, original_tln = mrcl_omniglot(classes=point) rln_saved = tf.keras.models.load_model("saved_models/rln_" + model_name) tln_saved = tf.keras.models.load_model("saved_models/tln_" + model_name) tln_weights = [ tln_saved.get_weights()[0], tln_saved.get_weights()[1], original_tln.get_weights()[2], original_tln.get_weights()[3] ] classification_parameters["online_learning_rate"] = lr rln, tln = mrcl_omniglot(classes=point) tln.set_weights(tln_weights) rln.set_weights(rln_saved.get_weights()) test_accuracy, train_accuracy = evaluate_classification_mrcl( evaluation_training_data, evaluation_test_data, rln, tln, point, classification_parameters) test_accuracy_results.append(test_accuracy) train_accuracy_results.append(train_accuracy) print( f"Learning rate {lr}, test accuracy {test_accuracy}, train accuracy {train_accuracy}" ) test_lr = lrs[np.argmax(np.array(test_accuracy_results))] train_lr = lrs[np.argmax(np.array(train_accuracy_results))] print( f"Number of classes {point}. Best testing learning rate is {test_lr} and best training learning rate is {train_lr}." ) test_accuracy_results = [] train_accuracy_results = [] print( f"Starting 50 iterations of evaluation testing with learning rate {test_lr}." ) for _ in range(50): classification_parameters["online_learning_rate"] = test_lr tf.keras.backend.clear_session() _, original_tln = mrcl_omniglot(classes=point) rln, tln = mrcl_omniglot(classes=point) rln_saved = tf.keras.models.load_model("saved_models/rln_" + model_name) tln.set_weights(original_tln.get_weights()) rln.set_weights(rln_saved.get_weights()) test_accuracy, _ = evaluate_classification_mrcl( evaluation_training_data, evaluation_test_data, rln, tln, point, classification_parameters) test_accuracy_results.append(str(test_accuracy)) with open(f"{save_dir}/{model_type}_omniglot_testing_{point}.json", 'w') as f: # writing JSON object json.dump(test_accuracy_results, f) print( f"Starting 50 iterations of evaluation training with learning rate {train_lr}." ) for _ in range(50): classification_parameters["online_learning_rate"] = train_lr tf.keras.backend.clear_session() _, original_tln = mrcl_omniglot(classes=point) rln_saved = tf.keras.models.load_model("saved_models/rln_" + model_name) rln, tln = mrcl_omniglot(classes=point) tln.set_weights(original_tln.get_weights()) rln.set_weights(rln_saved.get_weights()) _, train_accuracy = evaluate_classification_mrcl( evaluation_training_data, evaluation_test_data, rln, tln, point, classification_parameters) train_accuracy_results.append(str(train_accuracy)) with open(f"{save_dir}/{model_type}_omniglot_training_{point}.json", 'w') as f: # writing JSON object json.dump(train_accuracy_results, f)
import json from experiments.exp4_2.omniglot_model import mrcl_omniglot, get_eval_data_by_classes, evaluate_classification_mrcl from datasets.tf_datasets import load_omniglot from parameters import classification_parameters dataset = "omniglot" background_data, evaluation_data = load_omniglot(dataset, verbose=1) evaluation_training_data, evaluation_test_data = get_eval_data_by_classes(evaluation_data) current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") train_log_dir = 'logs/classification/gradient_tape/' + current_time + '/train' train_summary_writer = tf.summary.create_file_writer(train_log_dir) rln, tln = mrcl_omniglot(classes=200) try: os.stat("evaluation_results_scratch_omniglot") except: os.mkdir("evaluation_results_scratch_omniglot") points = [10, 50, 75, 100, 150, 200] for point in points: original_rln, original_tln = mrcl_omniglot(classes=point) lrs = [0.3, 0.1, 0.03, 0.01, 0.003, 0.001, 0.0003, 0.0001, 0.00003, 0.00001] test_accuracy_results = [] train_accuracy_results = [] for lr in lrs: classification_parameters["online_learning_rate"] = lr rln, tln = mrcl_omniglot(classes=point)
x_testing = [] y_testing = [] for class_id in range(len(background_training_data_5)): for training_item in background_training_data_5[class_id]: x_testing.append(training_item['image']) y_testing.append(training_item['label']) x_testing = tf.convert_to_tensor(x_testing) y_testing = tf.convert_to_tensor(y_testing) t = range(100) current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") learning_rates = [0.001, 0.0001, 0.00001] for lr in learning_rates: rln, tln = mrcl_omniglot() train_log_dir = f'logs/omniglot_{lr}/' + current_time + '/pre_train' train_summary_writer = tf.summary.create_file_writer(train_log_dir) for epoch, v in enumerate(t): for x, y in tf.data.Dataset.from_tensor_slices( (x_training, y_training)).shuffle(True).batch(32): loss, _ = pre_train(x, y, rln, tln, lr, pretraining_parameters) print("learning rate:", lr, "Epoch:", epoch, "Training loss:", loss.numpy()) with train_summary_writer.as_default(): tf.summary.scalar('Training loss', loss, step=epoch) if epoch % 10 == 0 and epoch != 0: total_correct = 0 for x, y in tf.data.Dataset.from_tensor_slices( (x_training, y_training)).shuffle(True).batch(32):