def test_model(model_name):
# load the model
model_path = os.path.join(MODELS_FOLDER, model_name)
model = torch.load(model_path)
model.eval()
model.to(device)
cm = torch.zeros(number_classes, number_classes)
for inputs, labels in dataloaders['test']:
inputs = inputs.to(device)
labels = labels.to(device)
with torch.set_grad_enabled(False):
# forward pass
outputs = model(inputs)
_, predictions = torch.max(outputs, 1)
# record performance in the confusion matrix
for t, p in zip(predictions.view(-1), labels.view(-1)):
cm[t.long(), p.long()] += 1
# record confusion matrix csv data
with open(STATUS_FILE, 'a') as gt:
# set print out precision
R = 4
writer = csv.writer(gt)
writer.writerow([
model_name,
round(mx.accuracy(cm).item(), R),
round(mx.sensitivity(cm, 0).item(), R),
round(mx.precision(cm, 0).item(), R),
round(mx.number_false_negative(cm, 0).item()),
round(mx.number_false_positive(cm, 0).item())
])
print("----------------------------------------")
print("Model name: ", model_name)
print("Confusion Matrix: \n", cm)
print("MX target sensitivity: ", mx.sensitivity(cm, 0))
print("MX class accuracy: ", mx.per_class_accuracy(cm))
print("----------------------------------------")
def model_selection(model, criterion, optimizer, scheduler, epochs=30):
# time execution
start = time.time()
# store best model weights
best_weights = copy.deepcopy(model.state_dict())
best_acc = 0.0
best_sen = 0.0
gt_file = os.path.join(LOG_DIR,PROGRESS_FILE)
for epoch in range(epochs):
print('Epoch: ',epoch)
for stage in ['train', 'val']:
if stage == 'train':
model.train()
else:
model.eval()
total_loss = 0.0
confusion_matrix = torch.zeros(number_classes, number_classes)
# loop through data
for inputs, labels in dataloaders[stage]:
inputs = inputs.to(device)
labels = labels.to(device)
# zero out gradients
optimizer.zero_grad()
# keep track of gradients for training only
with torch.set_grad_enabled(stage == 'train'):
# forward pass
outputs = model(inputs)
_, predictions = torch.max(outputs, 1)
# compute loss
loss = criterion(outputs, labels)
if stage == 'train':
loss.backward()
optimizer.step()
# record
# confusion matrix calculation
for t, p in zip(predictions.view(-1), labels.view(-1)):
confusion_matrix[t.long(), p.long()] += 1
total_loss += loss.item() * inputs.size(0)
if stage == 'train':
scheduler.step()
epoch_loss = total_loss / dataset_sizes[stage]
epoch_acc = mx.accuracy(confusion_matrix).item()
# get sensitivity for our target class 0
epoch_sen = mx.sensitivity(confusion_matrix,0).item()
# print out loss, acc, confusion matrix
print("Stage: ",stage," epoch loss: ", epoch_loss, " epoch acc: ", epoch_acc, " epoch sen: ", epoch_sen)
print(confusion_matrix)
# append to progress.csv for training
if stage == 'val':
with open(gt_file, 'a') as gt:
writer = csv.writer(gt)
if SELECT_ON_ACC:
writer.writerow([epoch, epoch_acc])
else:
writer.writerow([epoch, epoch_sen])
# when validating, check if current model is best
if SELECT_ON_ACC:
if stage == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_weights = copy.deepcopy(model.state_dict())
else:
if stage == 'val' and epoch_sen > best_sen:
best_sen = epoch_sen
best_weights = copy.deepcopy(model.state_dict())
spent = time.time() - start
print('Training and selection complete in {:.0f}m {:.0f}s'.format(spent // 60, spent % 60))
if SELECT_ON_ACC:
print('Best val accuracy: {:4f}'.format(best_acc))
else:
print('Best val sensitivity: {:4f}'.format(best_sen))
# return the best model
model.load_state_dict(best_weights)
return model
confusion_matrix = torch.zeros(number_classes, number_classes)
# loop through data
for inputs, labels in dataloaders['test']:
inputs = inputs.to(device)
labels = labels.to(device)
# keep track of gradients for training only
with torch.set_grad_enabled(False):
# forward pass
outputs = model(inputs)
_, predictions = torch.max(outputs, 1)
# compute loss
loss = criterion(outputs, labels)
# record
total_loss += loss.item() * inputs.size(0)
total_correct += torch.sum(predictions == labels.data)
# confusion matrix calculation
for t, p in zip(predictions.view(-1), labels.view(-1)):
confusion_matrix[t.long(), p.long()] += 1
print("Confusion Matrix: \n", confusion_matrix)
print("----------------------------------------")
print("MX accuracy: ", mx.accuracy(confusion_matrix))
print("MX target sensitivity: ", mx.sensitivity(confusion_matrix, 0))
print("MX class accuracy: ", mx.per_class_accuracy(confusion_matrix))
print("class names: ", class_names)
print("----------------------------------------")