def test(self, loader=None):
"""Test the model using the given loader and return test metrics."""
if loader is None:
loader = self.test_loader
t0 = time.time()
results = evaluate_model(model=self.model,
device=self.device,
loader=loader)
results["mean_accuracy"] = 100.0 * results["mean_accuracy"]
results.update({
"entropy":
float(self.entropy()),
"total_samples":
len(loader.sampler),
"non_zero_parameters":
count_nonzero_params(self.model)[1],
})
self.logger.info("testing duration: %s", time.time() - t0)
self.logger.info("mean_accuracy: %s", results["mean_accuracy"])
self.logger.info("mean_loss: %s", results["mean_loss"])
self.logger.info("entropy: %s", results["entropy"])
return results
def run_noise_tests(self, noise_values, loaders, epoch):
"""
Test the model with different noise values and return test metrics.
"""
ret = self.last_noise_results
# Just do noise tests every 3 iterations, about a 2X overall speedup
if epoch % 3 == 0 or ret is None:
ret = {"noise_values": noise_values, "noise_accuracies": []}
accuracy = 0.0
loss = 0.0
for _noise, loader in zip(noise_values, loaders):
test_result = evaluate_model(
model=self.model,
loader=loader,
device=self.device,
batches_in_epoch=self.test_batches_in_epoch,
criterion=self.loss_function,
)
accuracy += test_result["mean_accuracy"]
loss += test_result["mean_loss"]
ret["noise_accuracies"].append(test_result["mean_accuracy"])
ret["mean_accuracy"] = accuracy / len(noise_values)
ret["test_accuracy"] = ret["noise_accuracies"][0]
ret["noise_accuracy"] = ret["noise_accuracies"][-1]
ret["mean_loss"] = loss / len(noise_values)
self.last_noise_results = ret
return ret
def find_best_lr(self, num_classes_learned):
"""
This is a simple hyper-parameter search for a good lr:
1) Sample num_classes_learned classes
2) Train over the sampled classes; once for each lr
3) Evaluate the model on a held-out set
4) Repeat as many times as desired and pick the lr that performs the best
the most number times
"""
lr_all = []
# Grid search over lr
for _ in range(0, self.num_lr_search_runs):
# Choose num_classes_learned random classes to train and then test on.
new_tasks = np.random.choice(self.num_classes_eval,
num_classes_learned,
replace=False)
max_acc = -1000
for lr in self.lr_sweep_range:
# Reset output layer weights.
if self.reset_output_params:
output_params = self.get_named_output_params()
self.reset_params(output_params.values())
# Meta-test training.
test_train_param = self.get_named_test_train_params()
optim = Adam(test_train_param.values(), lr=lr)
for task in new_tasks:
self.test_train_loader.sampler.set_active_tasks(task)
train_model(
model=self.get_model(),
loader=self.test_train_loader,
optimizer=optim,
device=self.device,
criterion=self._loss_function,
)
# Meta-test testing.
self.test_test_loader.sampler.set_active_tasks(new_tasks)
results = evaluate_model(
model=self.get_model(),
loader=self.test_test_loader,
device=self.device,
criterion=self._loss_function,
)
correct = results["total_correct"]
acc = correct / len(self.test_test_loader.sampler.indices)
if (acc > max_acc):
max_acc = acc
max_lr = lr
lr_all.append(max_lr)
best_lr = float(stats.mode(lr_all)[0][0])
return best_lr
def test(self, loader=None):
if loader is None:
loader = self.test_loader
t0 = time.time()
results = evaluate_model(model=self.model,
device=self.device,
loader=loader,
batches_in_epoch=self.test_batches_in_epoch)
self.logger.info("testing duration: %s", time.time() - t0)
self.logger.info("mean_accuracy: %s", results["mean_accuracy"])
self.logger.info("mean_loss: %s", results["mean_loss"])
return results
def test(self, test_loader=None):
"""Test the model using the given loader and return test metrics."""
if test_loader is None:
test_loader = self.test_loader
ret = evaluate_model(self.model, test_loader, self.device)
ret["mean_accuracy"] = 100.0 * ret["mean_accuracy"]
entropy = self.entropy()
ret.update({
"entropy": float(entropy),
"total_samples": len(test_loader.sampler),
"non_zero_parameters": count_nonzero_params(self.model)[1],
})
return ret
def test(self, loader=None):
"""Test the model using the given loader and return test metrics."""
if loader is None:
loader = self.test_loader
t0 = time.time()
results = evaluate_model(model=self.model,
device=self.device,
loader=loader)
results.update({"entropy": float(self.entropy())})
self.logger.info("testing duration: %s", time.time() - t0)
self.logger.info("mean_accuracy: %s", results["mean_accuracy"])
self.logger.info("mean_loss: %s", results["mean_loss"])
self.logger.info("entropy: %s", results["entropy"])
return results
def _train(self):
if self._iteration == 0:
train_loader = self.first_loader
else:
train_loader = self.train_loader
train_model(
model=self.model,
loader=train_loader,
optimizer=self.optimizer,
device=self.device,
)
self.model.apply(rezero_weights)
self.model.apply(update_boost_strength)
return evaluate_model(model=self.model,
loader=self.test_loader,
device=self.device)
def _train(self):
if self._iteration == 0:
train_loader = self.first_loader
else:
train_loader = self.train_loader
train_model(
model=self.model,
loader=train_loader,
optimizer=self.optimizer,
device=self.device,
post_batch_callback=self._post_batch,
)
self.model.apply(update_boost_strength)
return evaluate_model(
model=self.model, loader=self.test_loader, device=self.device
)
def validate(self, loader=None):
if loader is None:
loader = self.val_loader
results = evaluate_model(
model=self.model,
loader=loader,
device=self.device,
criterion=self.loss_function,
batches_in_epoch=self.batches_in_epoch,
)
results.update(
batch_size=self.batch_size,
image_size=self.image_size,
learning_rate=self.get_lr()[0],
)
if self.rank == 0:
self.logger.info(results)
return results
def test(self, test_loader=None):
if test_loader is None:
test_loader = self.gen_test_loader
if not self.validation:
test_loader = test_loader
self.validation = False
else:
test_loader = self.validation_loader
ret = evaluate_model(self.model, test_loader, self.device)
ret["mean_accuracy"] = 100.0 * ret["mean_accuracy"]
entropy = self.entropy()
ret.update({
"entropy": float(entropy),
"total_samples": len(test_loader.sampler),
"non_zero_parameters": count_nonzero_params(self.model)[1],
})
return ret
def profile(model, num_workers, num_runs, batch_size, train=True):
# Load the training or test dataset and accompanying dataloader.
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
the_dataset = preprocessed_gsc(root=DATA_ROOT, train=train, download=False)
dataloader = DataLoader(
dataset=the_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers,
sampler=None,
pin_memory=torch.cuda.is_available(),
drop_last=False, # TODO: Will this affect timing?
)
# Verify accuracy of model
accuracies = evaluate_model(model, dataloader, device)
print("Accuracies:")
print(accuracies)
print(f"Profiling inference with batch size: {batch_size}")
run_times = []
samples_processed = []
for _ in range(num_runs + 1):
t0 = time()
s = run_model(model, dataloader, device)
run_times.append(time() - t0)
samples_processed.append(s)
# Compute and print statistics. We always ignore the first run to account for
# startup time
mu_time = np.mean(run_times[1:])
std_time = np.std(run_times[1:])
throughput = samples_processed[0] / mu_time
print(f"Run times: {mu_time:0.4f} ± {std_time:0.4f} "
f"averaged over {args.num_runs - 1} trials.")
print("All run times:", run_times)
print("Samples processed:", samples_processed)
print(f"Throughput words/sec: {throughput:0.2f} "
f"with batch size: {batch_size}")
return throughput, mu_time
def train_full(categorical=False):
experiment = get_experiment()
net = ToyNetwork(dpc=1, cnn_w_sparsity=0.1).cuda()
opt = torch.optim.SGD(net.parameters(), lr=0.1) # weight_decay=0.)
criterion = F.nll_loss
loader = experiment.full_train_loader
for x, y in loader:
opt.zero_grad()
if categorical:
out = net(x.cuda(), y.cuda())
else:
out = net(x.cuda()) # no categorical projection
loss = criterion(out, y.cuda())
loss.backward()
opt.step()
acc_ = evaluate_model(net, experiment.gen_test_loader,
torch.device("cuda"))["mean_accuracy"]
print("Accuracy: {}".format(np.round(acc_, 2)))
return acc_
def validate(self, epoch, loader=None):
if loader is None:
loader = self.val_loader
if epoch >= self.validate_after_epoch:
results = evaluate_model(
model=self.model,
loader=loader,
device=self.device,
criterion=self.loss_function,
batches_in_epoch=self.batches_in_epoch,
)
else:
results = {
"total_correct": 0,
"mean_loss": 0.0,
"mean_accuracy": 0.0,
}
results.update(learning_rate=self.get_lr()[0], )
self.logger.info(results)
return results
def run_meta_testing_phase(self, num_classes_learned):
"""
Run the meta-testing phase: train over num_classes_learned and then test over a
held-out set comprised of those same classes (aka the meta-test test set). This
shows the model's ability to conduct continual learning in a way that allows
generalization. As well, at the end of this phase, this function also evaluates
the models performance on the meta-test training set to evaluate it's ability to
memorize without forgetting.
"""
# Decide on the lr to use.
if self.run_lr_sweep:
lr = self.find_best_lr(num_classes_learned)
else:
lr = self.lr_sweep_range[-1]
meta_test_test_accuracies = []
meta_test_train_accuracies = []
for _ in range(0, self.num_meta_testing_runs):
# Choose num_classes_learned random classes to train and then test on.
new_tasks = np.random.choice(self.num_classes_eval,
num_classes_learned,
replace=False)
# Reset output layer weights.
if self.reset_output_params:
output_params = self.get_named_output_params()
self.reset_params(output_params.values())
# Meta-testing training.
test_train_param = self.get_named_test_train_params()
optim = Adam(test_train_param.values(), lr=lr)
for task in new_tasks:
self.test_train_loader.sampler.set_active_tasks(task)
train_model(
model=self.get_model(),
loader=self.test_train_loader,
optimizer=optim,
device=self.device,
criterion=self._loss_function,
)
# Meta-testing testing (using the test-test set).
self.test_test_loader.sampler.set_active_tasks(new_tasks)
results = evaluate_model(
model=self.model,
loader=self.test_test_loader,
device=self.device,
criterion=self._loss_function,
)
correct = results["total_correct"]
acc = correct / len(self.test_test_loader.sampler.indices)
meta_test_test_accuracies.append(acc)
# Meta-testing testing (using the test-train set).
self.test_train_eval_loader.sampler.set_active_tasks(new_tasks)
results = evaluate_model(
model=self.get_model(),
loader=self.test_train_eval_loader,
device=self.device,
criterion=self._loss_function,
)
correct = results["total_correct"]
acc = correct / len(self.test_train_eval_loader.sampler.indices)
meta_test_train_accuracies.append(acc)
return meta_test_train_accuracies, meta_test_test_accuracies, lr
def train_sequential(
categorical=False,
dpc=1,
cnn_weight_sparsity=0.1,
linear_w_sparsity=0.5,
cat_w_sparsity=0.01,
optim="Adam",
):
experiment = get_experiment()
net = ToyNetwork(
dpc=dpc,
cnn_w_sparsity=cnn_weight_sparsity,
linear_w_sparsity=linear_w_sparsity,
cat_w_sparsity=cat_w_sparsity,
).cuda()
if optim == "Adam":
opt = torch.optim.Adam(net.parameters(), lr=0.1, weight_decay=0.0)
else:
opt = torch.optim.SGD(net.parameters(), lr=0.1, weight_decay=0.0)
criterion = F.nll_loss
train_inds = np.arange(1, 5).reshape(2, 2)
losses = []
for i in range(len(train_inds)):
experiment.combine_classes(train_inds[i])
loader = experiment.train_loader
for x, y in loader:
opt.zero_grad()
if categorical:
out = net(x.cuda(), y.cuda())
else:
out = net(x.cuda()) # no categorical projection
loss = criterion(out, y.cuda())
loss.backward()
losses.append(loss.detach().cpu().numpy())
freeze_output_layer(net,
clear_labels(train_inds),
layer_type="kwinner",
linear_number="")
opt.step()
acc_ = [
np.round(
evaluate_model(net, experiment.test_loader[k],
torch.device("cuda"))["mean_accuracy"],
2,
) for k in train_inds[i]
]
print(acc_)
full_acc = [
np.round(
evaluate_model(net, experiment.test_loader[k],
torch.device("cuda"))["mean_accuracy"],
2,
) for k in train_inds.flatten()
]
print("Categorical: {}, acc={}".format(categorical, full_acc))
return full_acc
