def test(self, params, test_loader):
"""
Test the model using the given loader and return test metrics
"""
results = evaluateModel(model=self.model, device=self.device,
loader=test_loader)
entropy = self.model.entropy()
ret = {"num_correct": results["total_correct"],
"test_loss": results["loss"],
"testerror": results["accuracy"] * 100,
"entropy": float(entropy)}
return ret
def test(self, params, test_loader):
"""
Test the model using the given loader and return test metrics
"""
results = evaluateModel(model=self.model, device=self.device,
loader=test_loader)
entropy = self.model.entropy()
ret = {"num_correct": results["total_correct"],
"test_loss": results["loss"],
"testerror": results["accuracy"] * 100,
"entropy": float(entropy)}
return ret
def runNoiseTests(self, noiseValues, loaders):
"""
Test the model with different noise values and return test metrics.
"""
ret = {}
for noise, loader in zip(noiseValues, loaders):
testResult = evaluateModel(
model=self.model,
loader=loader,
device=self.device,
batches_in_epoch=self.test_batches_in_epoch,
criterion=self.loss_function)
ret[noise] = testResult
return ret
def iterate(self, params, repetition, iteration):
# Use 'iterations' to represent the task (0=[0-1], ..,5=[8-9])
task = iteration
position = self.cfgparser.sections().index(self.name) * 2
for epoch in tqdm.trange(self.epochs, position=position,
desc="{}:{}".format(self.name, task)):
if epoch == 0:
batch_size = self.first_epoch_batch_size
batches_in_epoch = self.batches_in_first_epoch
else:
batch_size = self.batch_size
batches_in_epoch = self.batches_in_epoch
# Train on a single task
train_loader = torch.utils.data.DataLoader(dataset=self.train_datasets[task],
batch_size=batch_size,
shuffle=True)
self.preEpoch()
trainModel(model=self.model, loader=train_loader,
optimizer=self.optimizer, device=self.device,
batches_in_epoch=batches_in_epoch,
criterion=self.loss_function,
progress_bar={"desc": "training", "position": position + 1})
self.postEpoch()
# Test on all trained tasks combined
combined_datasets = ConcatDataset(self.test_datasets[:task + 1])
test_loader = torch.utils.data.DataLoader(dataset=combined_datasets,
batch_size=self.test_batch_size,
shuffle=True)
return evaluateModel(model=self.model, device=self.device,
loader=test_loader,
criterion=self.loss_function,
progress={"desc": "testing", "position": position + 1})
def analyzeWeightPruning(args):
"""
Multiprocess function used to analyze the impact of nonzeros and accuracy
after pruning low weights and units with low dutycycle of a pre-trained model.
:param args: tuple with the following arguments:
- experiment path: The experiment results path
- configuration parameters: The parameters used in the experiment run
- minWeight: min weight to prune. If zero then no pruning
- minDutycycle: min threshold to prune. If less than zero then no pruning
- progress bar position:
When 'minWeight' is zero
:type args: tuple
:return: Panda DataFrame with the nonzero count for every weight variable in
the model and the evaluation results after the pruning the weights.
:rtype: :class:`pandas.DataFrame`
"""
path, params, minWeight, minDutycycle, position = args
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
datadir = os.path.join(params["datadir"], "speech_commands")
testDataDir = os.path.join(datadir, "test")
# Initialize speech test dataset for this experiment
n_mels = 32
testFeatureTransform = transforms.Compose([
FixAudioLength(),
ToMelSpectrogram(n_mels=n_mels),
ToTensor('mel_spectrogram', 'input'),
Unsqueeze(tensor_name="input", model_type=params["model_type"])
])
testDataset = SpeechCommandsDataset(
testDataDir,
testFeatureTransform,
silence_percentage=0,
)
test_loader = DataLoader(testDataset,
batch_size=params["batch_size"],
sampler=None,
shuffle=False)
# Load pre-trained model and evaluate with test dataset
model = torch.load(os.path.join(path, "model.pt"), map_location=device)
label = str(minWeight)
name = params["name"]
desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle)
model.pruneWeights(minWeight)
model.pruneDutycycles(minDutycycle)
# Collect nonzero
nonzero = {}
register_nonzero_counter(model, nonzero)
results = evaluateModel(model=model,
loader=test_loader,
device=device,
progress={
"desc": desc,
"position": position
})
unregister_counter_nonzero(model)
# Create table with results
table = pd.DataFrame.from_dict(nonzero)
noise_score = results["total_correct"]
table = table.assign(accuracy=results["accuracy"])
# Compute noise score
noise_values = tqdm(
[0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5],
position=position)
# Create noise dataset with noise transform
noiseTransform = transforms.Compose([
FixAudioLength(),
ToSTFT(),
ToMelSpectrogramFromSTFT(n_mels=n_mels),
DeleteSTFT(),
ToTensor('mel_spectrogram', 'input'),
Unsqueeze(tensor_name="input", model_type=params["model_type"])
])
noiseDataset = SpeechCommandsDataset(
testDataDir,
noiseTransform,
silence_percentage=0,
)
noise_loader = DataLoader(noiseDataset,
batch_size=params["batch_size"],
sampler=None,
shuffle=False)
for noise in noise_values:
noise_values.set_description("{}.noise({})".format(desc, noise))
xfrom = AddNoise(noise)
# Add noise to dataset transforms
noiseTransform.transforms.insert(1, xfrom)
# Evaluate model with noise
results = evaluateModel(model=model,
loader=noise_loader,
device=device)
# Remove noise from dataset transforms
noiseTransform.transforms.remove(xfrom)
# Update noise score
noise_score += results["total_correct"]
table = table.assign(noise_score=noise_score)
# Filter result for the 'weight' variable only
table = pd.DataFrame({label: table.xs("weight")})
table.drop(["input", "output"], inplace=True)
table.dropna(inplace=True)
return table
def analyzeWeightPruning(args):
"""
Multiprocess function used to analyze the impact of nonzeros and accuracy
after pruning low weights and units with low dutycycle of a pre-trained model.
:param args: tuple with the following arguments:
- experiment path: The experiment results path
- configuration parameters: The parameters used in the experiment run
- minWeight: min weight to prune. If zero then no pruning
- minDutycycle: min threshold to prune. If less than zero then no pruning
- progress bar position:
When 'minWeight' is zero
:type args: tuple
:return: Panda DataFrame with the nonzero count for every weight variable in
the model and the evaluation results after the pruning the weights.
:rtype: :class:`pandas.DataFrame`
"""
path, params, minWeight, minDutycycle, position = args
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Dataset transformations used during training. See mnist_sparse_experiment.py
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))])
# Initialize MNIST test dataset for this experiment
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(params["datadir"], train=False, download=True,
transform=transform),
batch_size=params["test_batch_size"], shuffle=True)
# Load pre-trained model and evaluate with test dataset
model = torch.load(os.path.join(path, "model.pt"), map_location=device)
label = str(minWeight)
name = params["name"]
desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle)
model.pruneWeights(minWeight)
model.pruneDutycycles(minDutycycle)
# Collect nonzero
nonzero = {}
register_nonzero_counter(model, nonzero)
results = evaluateModel(model=model, loader=test_loader, device=device,
progress={"desc": desc, "position": position})
unregister_counter_nonzero(model)
# Create table with results
table = pd.DataFrame.from_dict(nonzero)
noise_score = results["total_correct"]
table = table.assign(accuracy=results["accuracy"])
# Compute noise score
noise_values = tqdm([0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5],
position=position)
for noise in noise_values:
noise_values.set_description("{}.noise({})".format(desc, noise))
# Add noise to dataset transforms
transform.transforms.append(
RandomNoise(noise, whiteValue=0.1307 + 2 * 0.3081))
# Evaluate model with noise
results = evaluateModel(model=model, loader=test_loader, device=device)
# Remove noise from dataset transforms
transform.transforms.pop()
# Update noise score
noise_score += results["total_correct"]
table = table.assign(noise_score=noise_score)
# Filter result for the 'weight' variable only
table = pd.DataFrame({label: table.xs("weight")})
table.drop(["input", "output"], inplace=True)
table.dropna(inplace=True)
return table
def analyzeWeightPruning(args):
"""
Multiprocess function used to analyze the impact of nonzeros and accuracy
after pruning low weights and units with low dutycycle of a pre-trained model.
:param args: tuple with the following arguments:
- experiment path: The experiment results path
- configuration parameters: The parameters used in the experiment run
- minWeight: min weight to prune. If zero then no pruning
- minDutycycle: min threshold to prune. If less than zero then no pruning
- progress bar position:
When 'minWeight' is zero
:type args: tuple
:return: Panda DataFrame with the nonzero count for every weight variable in
the model and the evaluation results after the pruning the weights.
:rtype: :class:`pandas.DataFrame`
"""
path, params, minWeight, minDutycycle, position = args
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Dataset transformations used during training. See mnist_sparse_experiment.py
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))])
# Initialize MNIST test dataset for this experiment
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(params["datadir"], train=False, download=True,
transform=transform),
batch_size=params["test_batch_size"], shuffle=True)
# Load pre-trained model and evaluate with test dataset
model = torch.load(os.path.join(path, "model.pt"), map_location=device)
label = str(minWeight)
name = params["name"]
desc = "{}.minW({}).minD({})".format(name, minWeight, minDutycycle)
model.pruneWeights(minWeight)
model.pruneDutycycles(minDutycycle)
# Collect nonzero
nonzero = {}
register_nonzero_counter(model, nonzero)
results = evaluateModel(model=model, loader=test_loader, device=device,
progress={"desc": desc, "position": position})
unregister_counter_nonzero(model)
# Create table with results
table = pd.DataFrame.from_dict(nonzero)
noise_score = results["total_correct"]
table = table.assign(accuracy=results["accuracy"])
# Compute noise score
noise_values = tqdm([0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5],
position=position)
for noise in noise_values:
noise_values.set_description("{}.noise({})".format(desc, noise))
# Add noise to dataset transforms
transform.transforms.append(
RandomNoise(noise, whiteValue=0.1307 + 2 * 0.3081))
# Evaluate model with noise
results = evaluateModel(model=model, loader=test_loader, device=device)
# Remove noise from dataset transforms
transform.transforms.pop()
# Update noise score
noise_score += results["total_correct"]
table = table.assign(noise_score=noise_score)
# Filter result for the 'weight' variable only
table = pd.DataFrame({label: table.xs("weight")})
table.drop(["input", "output"], inplace=True)
table.dropna(inplace=True)
return table