def test(self, params, test_loader): """ Test the model using the given loader and return test metrics """ self.model.eval() test_loss = 0 correct = 0 nonzeros = None count_nonzeros = params.get("count_nonzeros", False) if count_nonzeros: nonzeros = {} register_nonzero_counter(self.model, nonzeros) with torch.no_grad(): for batch, target in test_loader: data = batch["input"] if params["model_type"] in ["resnet9", "cnn"]: data = torch.unsqueeze(data, 1) data, target = data.to(self.device), target.to(self.device) output = self.model(data) test_loss += F.nll_loss(output, target, reduction='sum').item() pred = output.max(1, keepdim=True)[1] correct += pred.eq(target.view_as(pred)).sum().item() # count nonzeros only once if count_nonzeros: count_nonzeros = False unregister_counter_nonzero(self.model) test_loss /= len(test_loader.sampler) test_error = 100. * correct / len(test_loader.sampler) entropy = self.model.entropy() ret = { "num_correct": correct, "test_loss": test_loss, "testerror": test_error, "entropy": float(entropy) } if nonzeros is not None: ret["nonzeros"] = nonzeros return ret
def analyzeWeightPruning(args): """ Multiprocess function used to analyze the impact of nonzeros and accuracy after pruning low weights and units with low dutycycle of a pre-trained model. :param args: tuple with the following arguments: - experiment path: The experiment results path - configuration parameters: The parameters used in the experiment run - minWeight: min weight to prune. If zero then no pruning - minDutycycle: min threshold to prune. If less than zero then no pruning - progress bar position: When 'minWeight' is zero :type args: tuple :return: Panda DataFrame with the nonzero count for every weight variable in the model and the evaluation results after the pruning the weights. :rtype: :class:`pandas.DataFrame` """ path, params, minWeight, minDutycycle, position = args device = torch.device("cuda" if torch.cuda.is_available() else "cpu") datadir = os.path.join(params["datadir"], "speech_commands") testDataDir = os.path.join(datadir, "test") # Initialize speech test dataset for this experiment n_mels = 32 testFeatureTransform = transforms.Compose([ FixAudioLength(), ToMelSpectrogram(n_mels=n_mels), ToTensor('mel_spectrogram', 'input'), Unsqueeze(tensor_name="input", model_type=params["model_type"]) ]) testDataset = SpeechCommandsDataset( testDataDir, testFeatureTransform, silence_percentage=0, ) test_loader = DataLoader(testDataset, batch_size=params["batch_size"], sampler=None, shuffle=False) # Load pre-trained model and evaluate with test dataset model = torch.load(os.path.join(path, "model.pt"), map_location=device) label = str(minWeight) name = params["name"] desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle) model.pruneWeights(minWeight) model.pruneDutycycles(minDutycycle) # Collect nonzero nonzero = {} register_nonzero_counter(model, nonzero) results = evaluateModel(model=model, loader=test_loader, device=device, progress={ "desc": desc, "position": position }) unregister_counter_nonzero(model) # Create table with results table = pd.DataFrame.from_dict(nonzero) noise_score = results["total_correct"] table = table.assign(accuracy=results["accuracy"]) # Compute noise score noise_values = tqdm( [0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5], position=position) # Create noise dataset with noise transform noiseTransform = transforms.Compose([ FixAudioLength(), ToSTFT(), ToMelSpectrogramFromSTFT(n_mels=n_mels), DeleteSTFT(), ToTensor('mel_spectrogram', 'input'), Unsqueeze(tensor_name="input", model_type=params["model_type"]) ]) noiseDataset = SpeechCommandsDataset( testDataDir, noiseTransform, silence_percentage=0, ) noise_loader = DataLoader(noiseDataset, batch_size=params["batch_size"], sampler=None, shuffle=False) for noise in noise_values: noise_values.set_description("{}.noise({})".format(desc, noise)) xfrom = AddNoise(noise) # Add noise to dataset transforms noiseTransform.transforms.insert(1, xfrom) # Evaluate model with noise results = evaluateModel(model=model, loader=noise_loader, device=device) # Remove noise from dataset transforms noiseTransform.transforms.remove(xfrom) # Update noise score noise_score += results["total_correct"] table = table.assign(noise_score=noise_score) # Filter result for the 'weight' variable only table = pd.DataFrame({label: table.xs("weight")}) table.drop(["input", "output"], inplace=True) table.dropna(inplace=True) return table
def analyzeWeightPruning(args): """ Multiprocess function used to analyze the impact of nonzeros and accuracy after pruning low weights and units with low dutycycle of a pre-trained model. :param args: tuple with the following arguments: - experiment path: The experiment results path - configuration parameters: The parameters used in the experiment run - minWeight: min weight to prune. If zero then no pruning - minDutycycle: min threshold to prune. If less than zero then no pruning - progress bar position: When 'minWeight' is zero :type args: tuple :return: Panda DataFrame with the nonzero count for every weight variable in the model and the evaluation results after the pruning the weights. :rtype: :class:`pandas.DataFrame` """ path, params, minWeight, minDutycycle, position = args device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Dataset transformations used during training. See mnist_sparse_experiment.py transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) # Initialize MNIST test dataset for this experiment test_loader = torch.utils.data.DataLoader( datasets.MNIST(params["datadir"], train=False, download=True, transform=transform), batch_size=params["test_batch_size"], shuffle=True) # Load pre-trained model and evaluate with test dataset model = torch.load(os.path.join(path, "model.pt"), map_location=device) label = str(minWeight) name = params["name"] desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle) model.pruneWeights(minWeight) model.pruneDutycycles(minDutycycle) # Collect nonzero nonzero = {} register_nonzero_counter(model, nonzero) results = evaluateModel(model=model, loader=test_loader, device=device, progress={"desc": desc, "position": position}) unregister_counter_nonzero(model) # Create table with results table = pd.DataFrame.from_dict(nonzero) noise_score = results["total_correct"] table = table.assign(accuracy=results["accuracy"]) # Compute noise score noise_values = tqdm([0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5], position=position) for noise in noise_values: noise_values.set_description("{}.noise({})".format(desc, noise)) # Add noise to dataset transforms transform.transforms.append( RandomNoise(noise, whiteValue=0.1307 + 2 * 0.3081)) # Evaluate model with noise results = evaluateModel(model=model, loader=test_loader, device=device) # Remove noise from dataset transforms transform.transforms.pop() # Update noise score noise_score += results["total_correct"] table = table.assign(noise_score=noise_score) # Filter result for the 'weight' variable only table = pd.DataFrame({label: table.xs("weight")}) table.drop(["input", "output"], inplace=True) table.dropna(inplace=True) return table