def test_model(model_name): # load the model model_path = os.path.join(MODELS_FOLDER, model_name) model = torch.load(model_path) model.eval() model.to(device) cm = torch.zeros(number_classes, number_classes) for inputs, labels in dataloaders['test']: inputs = inputs.to(device) labels = labels.to(device) with torch.set_grad_enabled(False): # forward pass outputs = model(inputs) _, predictions = torch.max(outputs, 1) # record performance in the confusion matrix for t, p in zip(predictions.view(-1), labels.view(-1)): cm[t.long(), p.long()] += 1 # record confusion matrix csv data with open(STATUS_FILE, 'a') as gt: # set print out precision R = 4 writer = csv.writer(gt) writer.writerow([ model_name, round(mx.accuracy(cm).item(), R), round(mx.sensitivity(cm, 0).item(), R), round(mx.precision(cm, 0).item(), R), round(mx.number_false_negative(cm, 0).item()), round(mx.number_false_positive(cm, 0).item()) ]) print("----------------------------------------") print("Model name: ", model_name) print("Confusion Matrix: \n", cm) print("MX target sensitivity: ", mx.sensitivity(cm, 0)) print("MX class accuracy: ", mx.per_class_accuracy(cm)) print("----------------------------------------")
def model_selection(model, criterion, optimizer, scheduler, epochs=30): # time execution start = time.time() # store best model weights best_weights = copy.deepcopy(model.state_dict()) best_acc = 0.0 best_sen = 0.0 gt_file = os.path.join(LOG_DIR,PROGRESS_FILE) for epoch in range(epochs): print('Epoch: ',epoch) for stage in ['train', 'val']: if stage == 'train': model.train() else: model.eval() total_loss = 0.0 confusion_matrix = torch.zeros(number_classes, number_classes) # loop through data for inputs, labels in dataloaders[stage]: inputs = inputs.to(device) labels = labels.to(device) # zero out gradients optimizer.zero_grad() # keep track of gradients for training only with torch.set_grad_enabled(stage == 'train'): # forward pass outputs = model(inputs) _, predictions = torch.max(outputs, 1) # compute loss loss = criterion(outputs, labels) if stage == 'train': loss.backward() optimizer.step() # record # confusion matrix calculation for t, p in zip(predictions.view(-1), labels.view(-1)): confusion_matrix[t.long(), p.long()] += 1 total_loss += loss.item() * inputs.size(0) if stage == 'train': scheduler.step() epoch_loss = total_loss / dataset_sizes[stage] epoch_acc = mx.accuracy(confusion_matrix).item() # get sensitivity for our target class 0 epoch_sen = mx.sensitivity(confusion_matrix,0).item() # print out loss, acc, confusion matrix print("Stage: ",stage," epoch loss: ", epoch_loss, " epoch acc: ", epoch_acc, " epoch sen: ", epoch_sen) print(confusion_matrix) # append to progress.csv for training if stage == 'val': with open(gt_file, 'a') as gt: writer = csv.writer(gt) if SELECT_ON_ACC: writer.writerow([epoch, epoch_acc]) else: writer.writerow([epoch, epoch_sen]) # when validating, check if current model is best if SELECT_ON_ACC: if stage == 'val' and epoch_acc > best_acc: best_acc = epoch_acc best_weights = copy.deepcopy(model.state_dict()) else: if stage == 'val' and epoch_sen > best_sen: best_sen = epoch_sen best_weights = copy.deepcopy(model.state_dict()) spent = time.time() - start print('Training and selection complete in {:.0f}m {:.0f}s'.format(spent // 60, spent % 60)) if SELECT_ON_ACC: print('Best val accuracy: {:4f}'.format(best_acc)) else: print('Best val sensitivity: {:4f}'.format(best_sen)) # return the best model model.load_state_dict(best_weights) return model
confusion_matrix = torch.zeros(number_classes, number_classes) # loop through data for inputs, labels in dataloaders['test']: inputs = inputs.to(device) labels = labels.to(device) # keep track of gradients for training only with torch.set_grad_enabled(False): # forward pass outputs = model(inputs) _, predictions = torch.max(outputs, 1) # compute loss loss = criterion(outputs, labels) # record total_loss += loss.item() * inputs.size(0) total_correct += torch.sum(predictions == labels.data) # confusion matrix calculation for t, p in zip(predictions.view(-1), labels.view(-1)): confusion_matrix[t.long(), p.long()] += 1 print("Confusion Matrix: \n", confusion_matrix) print("----------------------------------------") print("MX accuracy: ", mx.accuracy(confusion_matrix)) print("MX target sensitivity: ", mx.sensitivity(confusion_matrix, 0)) print("MX class accuracy: ", mx.per_class_accuracy(confusion_matrix)) print("class names: ", class_names) print("----------------------------------------")