def test(self, loader=None): """Test the model using the given loader and return test metrics.""" if loader is None: loader = self.test_loader t0 = time.time() results = evaluate_model(model=self.model, device=self.device, loader=loader) results["mean_accuracy"] = 100.0 * results["mean_accuracy"] results.update({ "entropy": float(self.entropy()), "total_samples": len(loader.sampler), "non_zero_parameters": count_nonzero_params(self.model)[1], }) self.logger.info("testing duration: %s", time.time() - t0) self.logger.info("mean_accuracy: %s", results["mean_accuracy"]) self.logger.info("mean_loss: %s", results["mean_loss"]) self.logger.info("entropy: %s", results["entropy"]) return results
def run_noise_tests(self, noise_values, loaders, epoch): """ Test the model with different noise values and return test metrics. """ ret = self.last_noise_results # Just do noise tests every 3 iterations, about a 2X overall speedup if epoch % 3 == 0 or ret is None: ret = {"noise_values": noise_values, "noise_accuracies": []} accuracy = 0.0 loss = 0.0 for _noise, loader in zip(noise_values, loaders): test_result = evaluate_model( model=self.model, loader=loader, device=self.device, batches_in_epoch=self.test_batches_in_epoch, criterion=self.loss_function, ) accuracy += test_result["mean_accuracy"] loss += test_result["mean_loss"] ret["noise_accuracies"].append(test_result["mean_accuracy"]) ret["mean_accuracy"] = accuracy / len(noise_values) ret["test_accuracy"] = ret["noise_accuracies"][0] ret["noise_accuracy"] = ret["noise_accuracies"][-1] ret["mean_loss"] = loss / len(noise_values) self.last_noise_results = ret return ret
def find_best_lr(self, num_classes_learned): """ This is a simple hyper-parameter search for a good lr: 1) Sample num_classes_learned classes 2) Train over the sampled classes; once for each lr 3) Evaluate the model on a held-out set 4) Repeat as many times as desired and pick the lr that performs the best the most number times """ lr_all = [] # Grid search over lr for _ in range(0, self.num_lr_search_runs): # Choose num_classes_learned random classes to train and then test on. new_tasks = np.random.choice(self.num_classes_eval, num_classes_learned, replace=False) max_acc = -1000 for lr in self.lr_sweep_range: # Reset output layer weights. if self.reset_output_params: output_params = self.get_named_output_params() self.reset_params(output_params.values()) # Meta-test training. test_train_param = self.get_named_test_train_params() optim = Adam(test_train_param.values(), lr=lr) for task in new_tasks: self.test_train_loader.sampler.set_active_tasks(task) train_model( model=self.get_model(), loader=self.test_train_loader, optimizer=optim, device=self.device, criterion=self._loss_function, ) # Meta-test testing. self.test_test_loader.sampler.set_active_tasks(new_tasks) results = evaluate_model( model=self.get_model(), loader=self.test_test_loader, device=self.device, criterion=self._loss_function, ) correct = results["total_correct"] acc = correct / len(self.test_test_loader.sampler.indices) if (acc > max_acc): max_acc = acc max_lr = lr lr_all.append(max_lr) best_lr = float(stats.mode(lr_all)[0][0]) return best_lr
def test(self, loader=None): if loader is None: loader = self.test_loader t0 = time.time() results = evaluate_model(model=self.model, device=self.device, loader=loader, batches_in_epoch=self.test_batches_in_epoch) self.logger.info("testing duration: %s", time.time() - t0) self.logger.info("mean_accuracy: %s", results["mean_accuracy"]) self.logger.info("mean_loss: %s", results["mean_loss"]) return results
def test(self, test_loader=None): """Test the model using the given loader and return test metrics.""" if test_loader is None: test_loader = self.test_loader ret = evaluate_model(self.model, test_loader, self.device) ret["mean_accuracy"] = 100.0 * ret["mean_accuracy"] entropy = self.entropy() ret.update({ "entropy": float(entropy), "total_samples": len(test_loader.sampler), "non_zero_parameters": count_nonzero_params(self.model)[1], }) return ret
def test(self, loader=None): """Test the model using the given loader and return test metrics.""" if loader is None: loader = self.test_loader t0 = time.time() results = evaluate_model(model=self.model, device=self.device, loader=loader) results.update({"entropy": float(self.entropy())}) self.logger.info("testing duration: %s", time.time() - t0) self.logger.info("mean_accuracy: %s", results["mean_accuracy"]) self.logger.info("mean_loss: %s", results["mean_loss"]) self.logger.info("entropy: %s", results["entropy"]) return results
def _train(self): if self._iteration == 0: train_loader = self.first_loader else: train_loader = self.train_loader train_model( model=self.model, loader=train_loader, optimizer=self.optimizer, device=self.device, ) self.model.apply(rezero_weights) self.model.apply(update_boost_strength) return evaluate_model(model=self.model, loader=self.test_loader, device=self.device)
def _train(self): if self._iteration == 0: train_loader = self.first_loader else: train_loader = self.train_loader train_model( model=self.model, loader=train_loader, optimizer=self.optimizer, device=self.device, post_batch_callback=self._post_batch, ) self.model.apply(update_boost_strength) return evaluate_model( model=self.model, loader=self.test_loader, device=self.device )
def validate(self, loader=None): if loader is None: loader = self.val_loader results = evaluate_model( model=self.model, loader=loader, device=self.device, criterion=self.loss_function, batches_in_epoch=self.batches_in_epoch, ) results.update( batch_size=self.batch_size, image_size=self.image_size, learning_rate=self.get_lr()[0], ) if self.rank == 0: self.logger.info(results) return results
def test(self, test_loader=None): if test_loader is None: test_loader = self.gen_test_loader if not self.validation: test_loader = test_loader self.validation = False else: test_loader = self.validation_loader ret = evaluate_model(self.model, test_loader, self.device) ret["mean_accuracy"] = 100.0 * ret["mean_accuracy"] entropy = self.entropy() ret.update({ "entropy": float(entropy), "total_samples": len(test_loader.sampler), "non_zero_parameters": count_nonzero_params(self.model)[1], }) return ret
def profile(model, num_workers, num_runs, batch_size, train=True): # Load the training or test dataset and accompanying dataloader. device = torch.device("cuda" if torch.cuda.is_available() else "cpu") the_dataset = preprocessed_gsc(root=DATA_ROOT, train=train, download=False) dataloader = DataLoader( dataset=the_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, sampler=None, pin_memory=torch.cuda.is_available(), drop_last=False, # TODO: Will this affect timing? ) # Verify accuracy of model accuracies = evaluate_model(model, dataloader, device) print("Accuracies:") print(accuracies) print(f"Profiling inference with batch size: {batch_size}") run_times = [] samples_processed = [] for _ in range(num_runs + 1): t0 = time() s = run_model(model, dataloader, device) run_times.append(time() - t0) samples_processed.append(s) # Compute and print statistics. We always ignore the first run to account for # startup time mu_time = np.mean(run_times[1:]) std_time = np.std(run_times[1:]) throughput = samples_processed[0] / mu_time print(f"Run times: {mu_time:0.4f} ± {std_time:0.4f} " f"averaged over {args.num_runs - 1} trials.") print("All run times:", run_times) print("Samples processed:", samples_processed) print(f"Throughput words/sec: {throughput:0.2f} " f"with batch size: {batch_size}") return throughput, mu_time
def train_full(categorical=False): experiment = get_experiment() net = ToyNetwork(dpc=1, cnn_w_sparsity=0.1).cuda() opt = torch.optim.SGD(net.parameters(), lr=0.1) # weight_decay=0.) criterion = F.nll_loss loader = experiment.full_train_loader for x, y in loader: opt.zero_grad() if categorical: out = net(x.cuda(), y.cuda()) else: out = net(x.cuda()) # no categorical projection loss = criterion(out, y.cuda()) loss.backward() opt.step() acc_ = evaluate_model(net, experiment.gen_test_loader, torch.device("cuda"))["mean_accuracy"] print("Accuracy: {}".format(np.round(acc_, 2))) return acc_
def validate(self, epoch, loader=None): if loader is None: loader = self.val_loader if epoch >= self.validate_after_epoch: results = evaluate_model( model=self.model, loader=loader, device=self.device, criterion=self.loss_function, batches_in_epoch=self.batches_in_epoch, ) else: results = { "total_correct": 0, "mean_loss": 0.0, "mean_accuracy": 0.0, } results.update(learning_rate=self.get_lr()[0], ) self.logger.info(results) return results
def run_meta_testing_phase(self, num_classes_learned): """ Run the meta-testing phase: train over num_classes_learned and then test over a held-out set comprised of those same classes (aka the meta-test test set). This shows the model's ability to conduct continual learning in a way that allows generalization. As well, at the end of this phase, this function also evaluates the models performance on the meta-test training set to evaluate it's ability to memorize without forgetting. """ # Decide on the lr to use. if self.run_lr_sweep: lr = self.find_best_lr(num_classes_learned) else: lr = self.lr_sweep_range[-1] meta_test_test_accuracies = [] meta_test_train_accuracies = [] for _ in range(0, self.num_meta_testing_runs): # Choose num_classes_learned random classes to train and then test on. new_tasks = np.random.choice(self.num_classes_eval, num_classes_learned, replace=False) # Reset output layer weights. if self.reset_output_params: output_params = self.get_named_output_params() self.reset_params(output_params.values()) # Meta-testing training. test_train_param = self.get_named_test_train_params() optim = Adam(test_train_param.values(), lr=lr) for task in new_tasks: self.test_train_loader.sampler.set_active_tasks(task) train_model( model=self.get_model(), loader=self.test_train_loader, optimizer=optim, device=self.device, criterion=self._loss_function, ) # Meta-testing testing (using the test-test set). self.test_test_loader.sampler.set_active_tasks(new_tasks) results = evaluate_model( model=self.model, loader=self.test_test_loader, device=self.device, criterion=self._loss_function, ) correct = results["total_correct"] acc = correct / len(self.test_test_loader.sampler.indices) meta_test_test_accuracies.append(acc) # Meta-testing testing (using the test-train set). self.test_train_eval_loader.sampler.set_active_tasks(new_tasks) results = evaluate_model( model=self.get_model(), loader=self.test_train_eval_loader, device=self.device, criterion=self._loss_function, ) correct = results["total_correct"] acc = correct / len(self.test_train_eval_loader.sampler.indices) meta_test_train_accuracies.append(acc) return meta_test_train_accuracies, meta_test_test_accuracies, lr
def train_sequential( categorical=False, dpc=1, cnn_weight_sparsity=0.1, linear_w_sparsity=0.5, cat_w_sparsity=0.01, optim="Adam", ): experiment = get_experiment() net = ToyNetwork( dpc=dpc, cnn_w_sparsity=cnn_weight_sparsity, linear_w_sparsity=linear_w_sparsity, cat_w_sparsity=cat_w_sparsity, ).cuda() if optim == "Adam": opt = torch.optim.Adam(net.parameters(), lr=0.1, weight_decay=0.0) else: opt = torch.optim.SGD(net.parameters(), lr=0.1, weight_decay=0.0) criterion = F.nll_loss train_inds = np.arange(1, 5).reshape(2, 2) losses = [] for i in range(len(train_inds)): experiment.combine_classes(train_inds[i]) loader = experiment.train_loader for x, y in loader: opt.zero_grad() if categorical: out = net(x.cuda(), y.cuda()) else: out = net(x.cuda()) # no categorical projection loss = criterion(out, y.cuda()) loss.backward() losses.append(loss.detach().cpu().numpy()) freeze_output_layer(net, clear_labels(train_inds), layer_type="kwinner", linear_number="") opt.step() acc_ = [ np.round( evaluate_model(net, experiment.test_loader[k], torch.device("cuda"))["mean_accuracy"], 2, ) for k in train_inds[i] ] print(acc_) full_acc = [ np.round( evaluate_model(net, experiment.test_loader[k], torch.device("cuda"))["mean_accuracy"], 2, ) for k in train_inds.flatten() ] print("Categorical: {}, acc={}".format(categorical, full_acc)) return full_acc