def test(self, params, test_loader):
"""
Test the model using the given loader and return test metrics
"""
self.model.eval()
test_loss = 0
correct = 0
nonzeros = None
count_nonzeros = params.get("count_nonzeros", False)
if count_nonzeros:
nonzeros = {}
register_nonzero_counter(self.model, nonzeros)
with torch.no_grad():
for batch, target in test_loader:
data = batch["input"]
if params["model_type"] in ["resnet9", "cnn"]:
data = torch.unsqueeze(data, 1)
data, target = data.to(self.device), target.to(self.device)
output = self.model(data)
test_loss += F.nll_loss(output, target, reduction='sum').item()
pred = output.max(1, keepdim=True)[1]
correct += pred.eq(target.view_as(pred)).sum().item()
# count nonzeros only once
if count_nonzeros:
count_nonzeros = False
unregister_counter_nonzero(self.model)
test_loss /= len(test_loader.sampler)
test_error = 100. * correct / len(test_loader.sampler)
entropy = self.model.entropy()
ret = {
"num_correct": correct,
"test_loss": test_loss,
"testerror": test_error,
"entropy": float(entropy)
}
if nonzeros is not None:
ret["nonzeros"] = nonzeros
return ret
def analyzeWeightPruning(args):
"""
Multiprocess function used to analyze the impact of nonzeros and accuracy
after pruning low weights and units with low dutycycle of a pre-trained model.
:param args: tuple with the following arguments:
- experiment path: The experiment results path
- configuration parameters: The parameters used in the experiment run
- minWeight: min weight to prune. If zero then no pruning
- minDutycycle: min threshold to prune. If less than zero then no pruning
- progress bar position:
When 'minWeight' is zero
:type args: tuple
:return: Panda DataFrame with the nonzero count for every weight variable in
the model and the evaluation results after the pruning the weights.
:rtype: :class:`pandas.DataFrame`
"""
path, params, minWeight, minDutycycle, position = args
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
datadir = os.path.join(params["datadir"], "speech_commands")
testDataDir = os.path.join(datadir, "test")
# Initialize speech test dataset for this experiment
n_mels = 32
testFeatureTransform = transforms.Compose([
FixAudioLength(),
ToMelSpectrogram(n_mels=n_mels),
ToTensor('mel_spectrogram', 'input'),
Unsqueeze(tensor_name="input", model_type=params["model_type"])
])
testDataset = SpeechCommandsDataset(
testDataDir,
testFeatureTransform,
silence_percentage=0,
)
test_loader = DataLoader(testDataset,
batch_size=params["batch_size"],
sampler=None,
shuffle=False)
# Load pre-trained model and evaluate with test dataset
model = torch.load(os.path.join(path, "model.pt"), map_location=device)
label = str(minWeight)
name = params["name"]
desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle)
model.pruneWeights(minWeight)
model.pruneDutycycles(minDutycycle)
# Collect nonzero
nonzero = {}
register_nonzero_counter(model, nonzero)
results = evaluateModel(model=model,
loader=test_loader,
device=device,
progress={
"desc": desc,
"position": position
})
unregister_counter_nonzero(model)
# Create table with results
table = pd.DataFrame.from_dict(nonzero)
noise_score = results["total_correct"]
table = table.assign(accuracy=results["accuracy"])
# Compute noise score
noise_values = tqdm(
[0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5],
position=position)
# Create noise dataset with noise transform
noiseTransform = transforms.Compose([
FixAudioLength(),
ToSTFT(),
ToMelSpectrogramFromSTFT(n_mels=n_mels),
DeleteSTFT(),
ToTensor('mel_spectrogram', 'input'),
Unsqueeze(tensor_name="input", model_type=params["model_type"])
])
noiseDataset = SpeechCommandsDataset(
testDataDir,
noiseTransform,
silence_percentage=0,
)
noise_loader = DataLoader(noiseDataset,
batch_size=params["batch_size"],
sampler=None,
shuffle=False)
for noise in noise_values:
noise_values.set_description("{}.noise({})".format(desc, noise))
xfrom = AddNoise(noise)
# Add noise to dataset transforms
noiseTransform.transforms.insert(1, xfrom)
# Evaluate model with noise
results = evaluateModel(model=model,
loader=noise_loader,
device=device)
# Remove noise from dataset transforms
noiseTransform.transforms.remove(xfrom)
# Update noise score
noise_score += results["total_correct"]
table = table.assign(noise_score=noise_score)
# Filter result for the 'weight' variable only
table = pd.DataFrame({label: table.xs("weight")})
table.drop(["input", "output"], inplace=True)
table.dropna(inplace=True)
return table
def analyzeWeightPruning(args):
"""
Multiprocess function used to analyze the impact of nonzeros and accuracy
after pruning low weights and units with low dutycycle of a pre-trained model.
:param args: tuple with the following arguments:
- experiment path: The experiment results path
- configuration parameters: The parameters used in the experiment run
- minWeight: min weight to prune. If zero then no pruning
- minDutycycle: min threshold to prune. If less than zero then no pruning
- progress bar position:
When 'minWeight' is zero
:type args: tuple
:return: Panda DataFrame with the nonzero count for every weight variable in
the model and the evaluation results after the pruning the weights.
:rtype: :class:`pandas.DataFrame`
"""
path, params, minWeight, minDutycycle, position = args
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Dataset transformations used during training. See mnist_sparse_experiment.py
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))])
# Initialize MNIST test dataset for this experiment
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(params["datadir"], train=False, download=True,
transform=transform),
batch_size=params["test_batch_size"], shuffle=True)
# Load pre-trained model and evaluate with test dataset
model = torch.load(os.path.join(path, "model.pt"), map_location=device)
label = str(minWeight)
name = params["name"]
desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle)
model.pruneWeights(minWeight)
model.pruneDutycycles(minDutycycle)
# Collect nonzero
nonzero = {}
register_nonzero_counter(model, nonzero)
results = evaluateModel(model=model, loader=test_loader, device=device,
progress={"desc": desc, "position": position})
unregister_counter_nonzero(model)
# Create table with results
table = pd.DataFrame.from_dict(nonzero)
noise_score = results["total_correct"]
table = table.assign(accuracy=results["accuracy"])
# Compute noise score
noise_values = tqdm([0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5],
position=position)
for noise in noise_values:
noise_values.set_description("{}.noise({})".format(desc, noise))
# Add noise to dataset transforms
transform.transforms.append(
RandomNoise(noise, whiteValue=0.1307 + 2 * 0.3081))
# Evaluate model with noise
results = evaluateModel(model=model, loader=test_loader, device=device)
# Remove noise from dataset transforms
transform.transforms.pop()
# Update noise score
noise_score += results["total_correct"]
table = table.assign(noise_score=noise_score)
# Filter result for the 'weight' variable only
table = pd.DataFrame({label: table.xs("weight")})
table.drop(["input", "output"], inplace=True)
table.dropna(inplace=True)
return table
def analyzeWeightPruning(args):
"""
Multiprocess function used to analyze the impact of nonzeros and accuracy
after pruning low weights and units with low dutycycle of a pre-trained model.
:param args: tuple with the following arguments:
- experiment path: The experiment results path
- configuration parameters: The parameters used in the experiment run
- minWeight: min weight to prune. If zero then no pruning
- minDutycycle: min threshold to prune. If less than zero then no pruning
- progress bar position:
When 'minWeight' is zero
:type args: tuple
:return: Panda DataFrame with the nonzero count for every weight variable in
the model and the evaluation results after the pruning the weights.
:rtype: :class:`pandas.DataFrame`
"""
path, params, minWeight, minDutycycle, position = args
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Dataset transformations used during training. See mnist_sparse_experiment.py
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))])
# Initialize MNIST test dataset for this experiment
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(params["datadir"], train=False, download=True,
transform=transform),
batch_size=params["test_batch_size"], shuffle=True)
# Load pre-trained model and evaluate with test dataset
model = torch.load(os.path.join(path, "model.pt"), map_location=device)
label = str(minWeight)
name = params["name"]
desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle)
model.pruneWeights(minWeight)
model.pruneDutycycles(minDutycycle)
# Collect nonzero
nonzero = {}
register_nonzero_counter(model, nonzero)
results = evaluateModel(model=model, loader=test_loader, device=device,
progress={"desc": desc, "position": position})
unregister_counter_nonzero(model)
# Create table with results
table = pd.DataFrame.from_dict(nonzero)
noise_score = results["total_correct"]
table = table.assign(accuracy=results["accuracy"])
# Compute noise score
noise_values = tqdm([0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5],
position=position)
for noise in noise_values:
noise_values.set_description("{}.noise({})".format(desc, noise))
# Add noise to dataset transforms
transform.transforms.append(
RandomNoise(noise, whiteValue=0.1307 + 2 * 0.3081))
# Evaluate model with noise
results = evaluateModel(model=model, loader=test_loader, device=device)
# Remove noise from dataset transforms
transform.transforms.pop()
# Update noise score
noise_score += results["total_correct"]
table = table.assign(noise_score=noise_score)
# Filter result for the 'weight' variable only
table = pd.DataFrame({label: table.xs("weight")})
table.drop(["input", "output"], inplace=True)
table.dropna(inplace=True)
return table