def run_experiment(config):
exp_class = config["experiment_class"]
exp = exp_class()
exp.setup_experiment(config)
exp.model = exp.model.to(exp.device)
# --------------------------- CONTINUAL LEARNING PHASE -------------------------- #
for task_id in range(num_tasks):
# Train model on current task
t1 = time.time()
exp.train_loader.sampler.set_active_tasks(task_id)
for _ in range(num_epochs):
train_model(exp)
t2 = time.time()
print(f"train time [task {task_id}]: {t2 - t1}")
# Evaluate model accuracy on each task separately
if task_id in config["epochs_to_validate"]:
print(f"\n=== AFTER TASK {task_id} ===\n")
for eval_task_id in range(task_id + 1):
exp.val_loader.sampler.set_active_tasks(eval_task_id)
acc_task = evaluate_model(exp)
if isinstance(acc_task, tuple):
acc_task = acc_task[0]
print(f"task {eval_task_id} accuracy: {acc_task}")
t3 = time.time()
print(f"\nevaluation time: {t3 - t2}")
print(f"====================\n")
# ------------------------------------------------------------------------------- #
# Report final aggregate accuracy
exp.val_loader.sampler.set_active_tasks(range(num_tasks))
acc_task = evaluate_model(exp)
if isinstance(acc_task, tuple):
acc_task = acc_task[0]
print(f"Total test accuracy: {acc_task}")
# Print entropy of layers
max_possible_entropy = max_entropy(exp.model.hidden_size,
int(0.05 * exp.model.hidden_size))
if exp.model.kw:
print(f" KW1 entropy: {exp.model.kw1.entropy().item()}")
print(f" KW2 entropy: {exp.model.kw2.entropy().item()}")
print(f" max entropy: {max_possible_entropy}")
print("")
def max_entropy(self):
"""Returns the maximum total entropy we can expect from this layer."""
return max_entropy(self.n, int(self.n * self.percent_on))
