def createTestLoaders(self, noise_values):
"""
Create a list of data loaders, one for each noise value
"""
print("Creating test loaders for noise values:", noise_values)
loaders = []
for noise in noise_values:
transform_noise_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
RandomNoise(noise,
whiteValue=0.5 + 2 * 0.20,
blackValue=0.5 - 2 * 0.2),
])
testset = datasets.CIFAR10(root=self.dataDir,
train=False,
download=True,
transform=transform_noise_test)
loaders.append(
DataLoader(testset,
batch_size=self.test_batch_size,
shuffle=False))
return loaders
def runNoiseTests(self, params):
"""
Test the model with different noise values and return test metrics.
"""
ret = {}
# Noise on validation data
validation = {} if self.validation_sampler is not None else None
# Test with noise
total_correct = 0
validation_total_correct = 0
for noise in [
0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5
]:
transform = transforms.Compose([
transforms.ToTensor(),
RandomNoise(noise, whiteValue=0.1307 + 2 * 0.3081),
transforms.Normalize((0.1307, ), (0.3081, ))
])
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(self.dataDir, train=False, transform=transform),
batch_size=params["test_batch_size"],
shuffle=True)
testResult = self.test(params, test_loader)
total_correct += testResult["num_correct"]
ret[noise] = testResult
if validation is not None:
validation_loader = torch.utils.data.DataLoader(
datasets.MNIST(self.dataDir,
train=True,
transform=transform),
sampler=self.validation_sampler,
batch_size=params["test_batch_size"])
validationResult = self.test(params, validation_loader)
validation_total_correct += validationResult["num_correct"]
validation[noise] = validationResult
ret["totalCorrect"] = total_correct
ret["testerror"] = ret[0.0]["testerror"]
ret["entropy"] = ret[0.0]["entropy"]
if "nonzeros" in ret[0.0]:
ret["nonzeros"] = ret[0.0]["nonzeros"]
if validation is not None:
validation["totalCorrect"] = validation_total_correct
validation["testerror"] = validation[0.0]["testerror"]
validation["entropy"] = validation[0.0]["entropy"]
ret["validation"] = validation
return ret
def analyzeWeightPruning(args):
"""
Multiprocess function used to analyze the impact of nonzeros and accuracy
after pruning low weights and units with low dutycycle of a pre-trained model.
:param args: tuple with the following arguments:
- experiment path: The experiment results path
- configuration parameters: The parameters used in the experiment run
- minWeight: min weight to prune. If zero then no pruning
- minDutycycle: min threshold to prune. If less than zero then no pruning
- progress bar position:
When 'minWeight' is zero
:type args: tuple
:return: Panda DataFrame with the nonzero count for every weight variable in
the model and the evaluation results after the pruning the weights.
:rtype: :class:`pandas.DataFrame`
"""
path, params, minWeight, minDutycycle, position = args
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Dataset transformations used during training. See mnist_sparse_experiment.py
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))])
# Initialize MNIST test dataset for this experiment
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(params["datadir"], train=False, download=True,
transform=transform),
batch_size=params["test_batch_size"], shuffle=True)
# Load pre-trained model and evaluate with test dataset
model = torch.load(os.path.join(path, "model.pt"), map_location=device)
label = str(minWeight)
name = params["name"]
desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle)
model.pruneWeights(minWeight)
model.pruneDutycycles(minDutycycle)
# Collect nonzero
nonzero = {}
register_nonzero_counter(model, nonzero)
results = evaluateModel(model=model, loader=test_loader, device=device,
progress={"desc": desc, "position": position})
unregister_counter_nonzero(model)
# Create table with results
table = pd.DataFrame.from_dict(nonzero)
noise_score = results["total_correct"]
table = table.assign(accuracy=results["accuracy"])
# Compute noise score
noise_values = tqdm([0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5],
position=position)
for noise in noise_values:
noise_values.set_description("{}.noise({})".format(desc, noise))
# Add noise to dataset transforms
transform.transforms.append(
RandomNoise(noise, whiteValue=0.1307 + 2 * 0.3081))
# Evaluate model with noise
results = evaluateModel(model=model, loader=test_loader, device=device)
# Remove noise from dataset transforms
transform.transforms.pop()
# Update noise score
noise_score += results["total_correct"]
table = table.assign(noise_score=noise_score)
# Filter result for the 'weight' variable only
table = pd.DataFrame({label: table.xs("weight")})
table.drop(["input", "output"], inplace=True)
table.dropna(inplace=True)
return table