def pretrain(sort_samples=True, model_name="mrcl"):
print(f"GPU is available: {tf.test.is_gpu_available()}")
background_data, _ = load_omniglot(verbose=1)
background_training_data, _, _ = get_background_data_by_classes(background_data, sort=sort_samples)
s_learn, s_remember = partition_into_disjoint(background_training_data)
rln, tln = mrcl_omniglot()
t = range(15000)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = 'logs/classification/pretraining/omniglot/' + model_name + '/gradient_tape/' + current_time + '/train'
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
tln_initial = tf.keras.models.clone_model(tln)
for epoch, v in enumerate(t):
x_rand, y_rand = sample_random_10_classes(s_remember, background_training_data)
x_traj, y_traj = sample_trajectory(s_learn, background_training_data)
loss = pretrain_classification_mrcl(x_traj, y_traj, x_rand, y_rand, rln, tln, tln_initial, classification_parameters)
# Check metrics
rep = rln(x_rand)
rep = np.array(rep)
counts = np.isclose(rep, 0).sum(axis=1) / rep.shape[1]
sparsity = np.mean(counts)
with train_summary_writer.as_default():
tf.summary.scalar('Sparsity', sparsity, step=epoch)
tf.summary.scalar('Training loss', loss, step=epoch)
if (epoch+1) % 1000 == 0:
print("Epoch:", epoch, "Sparsity:", sparsity, "Training loss:", loss.numpy())
save_models(tln, f"tln_pretraining_{model_name}_{epoch}_omniglot")
save_models(rln, f"rln_pretraining_{model_name}_{epoch}_omniglot")
def build_omniglot_model(self,
n_layers_rln=6,
n_layers_tln=2,
filters=256,
hidden_units_per_layer=300,
seed=None):
self.model_rln, self.model_tln = mrcl_omniglot(n_layers_rln,
n_layers_tln, filters,
hidden_units_per_layer,
seed)
self.compute_loss_training = tf.function(self._compute_loss)
self.compute_loss_no_training = tf.function(
self._compute_loss_no_regularization)
def evaluate(model_name, model_type="mrcl"):
_, evaluation_data = load_omniglot(verbose=1)
evaluation_training_data, evaluation_test_data = get_eval_data_by_classes(
evaluation_data)
save_dir = "results/omniglot/" + model_type
try:
os.stat(save_dir)
except IOError:
os.mkdir(save_dir)
points = [10, 50, 75, 100, 150, 200]
for point in points:
lrs = [
0.3, 0.1, 0.03, 0.01, 0.003, 0.001, 0.0003, 0.0001, 0.00003,
0.00001
]
test_accuracy_results = []
train_accuracy_results = []
for lr in lrs:
tf.keras.backend.clear_session()
_, original_tln = mrcl_omniglot(classes=point)
rln_saved = tf.keras.models.load_model("saved_models/rln_" +
model_name)
tln_saved = tf.keras.models.load_model("saved_models/tln_" +
model_name)
tln_weights = [
tln_saved.get_weights()[0],
tln_saved.get_weights()[1],
original_tln.get_weights()[2],
original_tln.get_weights()[3]
]
classification_parameters["online_learning_rate"] = lr
rln, tln = mrcl_omniglot(classes=point)
tln.set_weights(tln_weights)
rln.set_weights(rln_saved.get_weights())
test_accuracy, train_accuracy = evaluate_classification_mrcl(
evaluation_training_data, evaluation_test_data, rln, tln,
point, classification_parameters)
test_accuracy_results.append(test_accuracy)
train_accuracy_results.append(train_accuracy)
print(
f"Learning rate {lr}, test accuracy {test_accuracy}, train accuracy {train_accuracy}"
)
test_lr = lrs[np.argmax(np.array(test_accuracy_results))]
train_lr = lrs[np.argmax(np.array(train_accuracy_results))]
print(
f"Number of classes {point}. Best testing learning rate is {test_lr} and best training learning rate is {train_lr}."
)
test_accuracy_results = []
train_accuracy_results = []
print(
f"Starting 50 iterations of evaluation testing with learning rate {test_lr}."
)
for _ in range(50):
classification_parameters["online_learning_rate"] = test_lr
tf.keras.backend.clear_session()
_, original_tln = mrcl_omniglot(classes=point)
rln, tln = mrcl_omniglot(classes=point)
rln_saved = tf.keras.models.load_model("saved_models/rln_" +
model_name)
tln.set_weights(original_tln.get_weights())
rln.set_weights(rln_saved.get_weights())
test_accuracy, _ = evaluate_classification_mrcl(
evaluation_training_data, evaluation_test_data, rln, tln,
point, classification_parameters)
test_accuracy_results.append(str(test_accuracy))
with open(f"{save_dir}/{model_type}_omniglot_testing_{point}.json",
'w') as f: # writing JSON object
json.dump(test_accuracy_results, f)
print(
f"Starting 50 iterations of evaluation training with learning rate {train_lr}."
)
for _ in range(50):
classification_parameters["online_learning_rate"] = train_lr
tf.keras.backend.clear_session()
_, original_tln = mrcl_omniglot(classes=point)
rln_saved = tf.keras.models.load_model("saved_models/rln_" +
model_name)
rln, tln = mrcl_omniglot(classes=point)
tln.set_weights(original_tln.get_weights())
rln.set_weights(rln_saved.get_weights())
_, train_accuracy = evaluate_classification_mrcl(
evaluation_training_data, evaluation_test_data, rln, tln,
point, classification_parameters)
train_accuracy_results.append(str(train_accuracy))
with open(f"{save_dir}/{model_type}_omniglot_training_{point}.json",
'w') as f: # writing JSON object
json.dump(train_accuracy_results, f)
import json
from experiments.exp4_2.omniglot_model import mrcl_omniglot, get_eval_data_by_classes, evaluate_classification_mrcl
from datasets.tf_datasets import load_omniglot
from parameters import classification_parameters
dataset = "omniglot"
background_data, evaluation_data = load_omniglot(dataset, verbose=1)
evaluation_training_data, evaluation_test_data = get_eval_data_by_classes(evaluation_data)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = 'logs/classification/gradient_tape/' + current_time + '/train'
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
rln, tln = mrcl_omniglot(classes=200)
try:
os.stat("evaluation_results_scratch_omniglot")
except:
os.mkdir("evaluation_results_scratch_omniglot")
points = [10, 50, 75, 100, 150, 200]
for point in points:
original_rln, original_tln = mrcl_omniglot(classes=point)
lrs = [0.3, 0.1, 0.03, 0.01, 0.003, 0.001, 0.0003, 0.0001, 0.00003, 0.00001]
test_accuracy_results = []
train_accuracy_results = []
for lr in lrs:
classification_parameters["online_learning_rate"] = lr
rln, tln = mrcl_omniglot(classes=point)
x_testing = []
y_testing = []
for class_id in range(len(background_training_data_5)):
for training_item in background_training_data_5[class_id]:
x_testing.append(training_item['image'])
y_testing.append(training_item['label'])
x_testing = tf.convert_to_tensor(x_testing)
y_testing = tf.convert_to_tensor(y_testing)
t = range(100)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
learning_rates = [0.001, 0.0001, 0.00001]
for lr in learning_rates:
rln, tln = mrcl_omniglot()
train_log_dir = f'logs/omniglot_{lr}/' + current_time + '/pre_train'
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
for epoch, v in enumerate(t):
for x, y in tf.data.Dataset.from_tensor_slices(
(x_training, y_training)).shuffle(True).batch(32):
loss, _ = pre_train(x, y, rln, tln, lr, pretraining_parameters)
print("learning rate:", lr, "Epoch:", epoch, "Training loss:",
loss.numpy())
with train_summary_writer.as_default():
tf.summary.scalar('Training loss', loss, step=epoch)
if epoch % 10 == 0 and epoch != 0:
total_correct = 0
for x, y in tf.data.Dataset.from_tensor_slices(
(x_training, y_training)).shuffle(True).batch(32):