def load_arrinet_classifier(isprocessed=False, ismultispectral=True): """Classifier for tissue types of image patches Dependencies: get_modelpath """ # Get model path and number of channels modelpath, n_channels = get_modelpath(isprocessed=isprocessed, ismultispectral=ismultispectral) print('Loading model path: ', modelpath) # Initialize Arrinet net = densenet_av.densenet_40_12_bc(in_channels=n_channels) net.fc = nn.Linear(net.fc.in_features, NUM_CLASSES) # adjust last fully-connected layer dimensions # net = net.to(device) net.load_state_dict(torch.load(modelpath)) net.eval() return net
def main(): # Get start time for this run timestr = time.strftime("%Y%m%d-%H%M%S") print(timestr) # for tt in range(1): for tt in range(30): # include this for loop to randomly sample learning rate, other hyperparameters # Get start time for this run timestr = time.strftime("%Y%m%d-%H%M%S") print(timestr) xx = 3 + random.random()*1 LEARNING_RATE = 10**-xx # random search hyperparameters yy = 3 + random.random()*1 ALPHA_L2REG = 1*10**-yy zz = random.random()*0 DROPOUT_RATE = zz print('Iteration: ', tt, LEARNING_RATE , ALPHA_L2REG, DROPOUT_RATE) for xx in [True]: # for xx in [True, False]: ISMULTISPECTRAL = xx #%% Data loading if ISMULTISPECTRAL: # 21-channel pickled tiled images # set paths to RGB images data_dir = PATH_PATCHES3D out_path = PATH_OUTPUT3D # Data augmentation and normalization for training # Just normalization for validation data_transforms = { 'train': transforms.Compose([ transforms.ToTensor(), transforms.Normalize(MEAN_CHANNEL_PIXELVALS, STD_CHANNEL_PIXELVALS) ]), 'val': transforms.Compose([ transforms.ToTensor(), transforms.Normalize(MEAN_CHANNEL_PIXELVALS, STD_CHANNEL_PIXELVALS) ]), 'test': transforms.Compose([ transforms.ToTensor(), transforms.Normalize(MEAN_CHANNEL_PIXELVALS, STD_CHANNEL_PIXELVALS) ]), } num_channels = 21 - N_REMOVED else: # RGB 3-channel, pickled images # set paths to RGB images data_dir = PATH_PATCHES2D out_path = PATH_OUTPUT2D # Data augmentation and normalization for training # Just normalization for validation data_transforms = { 'train': transforms.Compose([ transforms.ToTensor(), transforms.Normalize(MEAN_CHANNEL_PIXELVALS[:3], STD_CHANNEL_PIXELVALS[:3]) ]), 'val': transforms.Compose([ transforms.ToTensor(), transforms.Normalize(MEAN_CHANNEL_PIXELVALS[:3], STD_CHANNEL_PIXELVALS[:3]) ]), 'test': transforms.Compose([ transforms.ToTensor(), transforms.Normalize(MEAN_CHANNEL_PIXELVALS[:3], STD_CHANNEL_PIXELVALS[:3]) ]), } num_channels = 3 image_datasets = {x: datasets.DatasetFolder(os.path.join(data_dir, x), loader=pickle_loader, extensions='.pkl', transform=data_transforms[x]) for x in ['train', 'val', 'test']} print('Num channels', num_channels) if not LOADMODEL: dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=BATCH_SIZE, shuffle=True, num_workers=0) for x in ['train', 'val', 'test']} else: dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=BATCH_SIZE, shuffle=False, num_workers=0) for x in ['train', 'val', 'test']} dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val', 'test']} class_names = image_datasets['train'].classes num_classes = len(class_names) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print('Dataset sizes', dataset_sizes) print('Number of classes', num_classes) print('GPU vs CPU:', device) #%%Finetuning the convnet #Load a pretrained model and reset final fully connected layer. # model_ft = models.resnet18(pretrained=True) model_ft = densenet_av.densenet_40_12_bc(pretrained=ISPRETRAINED, in_channels=num_channels, drop_rate=DROPOUT_RATE) num_ftrs = model_ft.fc.in_features print('model_ft.fc.in_features =', num_ftrs) #debugging model_ft.fc = nn.Linear(num_ftrs, num_classes) model_ft = model_ft.to(device) criterion = nn.CrossEntropyLoss() # Observe that all parameters are being optimized # optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9) optimizer_ft = optim.Adam(model_ft.parameters(), lr=LEARNING_RATE, weight_decay=ALPHA_L2REG) # defaulat ADAM lr = 0.001 # Decay LR by a factor of 0.1 every 7 epochs exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=LRDECAY_STEP, gamma=LRDECAY_GAMMA) # exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=1000, gamma=LRDECAY_GAMMA) # For Adam optimizer, no need for LR decay #%% Train and evaluate if LOADMODEL: # load weights instead of training print('Loading model... Select loss/acc file:') filepath = mat.uigetfile() [cache_loss, cache_acc] = pickle.load(open(filepath, "rb")) print('Loading model... ') modelpath = filepath.replace('lossacc', 'modelparam').replace('.pkl', '.pt') model_ft.load_state_dict(torch.load(modelpath)) model_ft.eval() # Get same filename for saving path_head, path_tail = os.path.split(filepath) filename_pre, path_ext = os.path.splitext(path_tail) # # 8-25-2019 # # Obtain per-image classification accuracy based on patches - loop through folders without dataloader # for phase in ['train', 'val', 'test']: # data_dir2 = os.path.join(data_dir, phase) # tissues = os.listdir(data_dir2) # should be num_classes # of folders # print('Evaluating per-specimen accuracy on dataset: ', phase) # # # Iterate over tissue classes # for tt, tissue in enumerate(tissues): # tissue_folder = os.path.join(data_dir2, tissue) # tissue_files = os.listdir(tissue_folder) # tissue_dates = [i.split('_', 1)[0] for i in tissue_files] # unique_dates = list(set(tissue_dates)) ## print(unique_dates) # num_dates = np.size(unique_dates) # # num_patches_tissue_date = np.zeros((num_dates, 1)) # num_correctpatches_tissue_date = np.zeros((num_dates, 1)) # iscorrect_tissue_date = np.zeros((num_dates, 1)) # # # Calculate fraction of correct patch predictions per tissue-date specimen # num_patches = 0 # for i, session in enumerate(unique_dates): ## print(session) # num_patches_tissue_date[i] = tissue_dates.count(session) # tissue_patches_session_filenames = [item for item in tissue_files if item.startswith(session)] # # # Load patches into one batch of shape [M, C, H, W] # # where M is batch size (# patches), C is # channels # patches_session = np.zeros((int(num_patches_tissue_date[i]), num_channels, TILE_HEIGHT, TILE_WIDTH)) # for j, patch_filename in enumerate(tissue_patches_session_filenames): # if ISMULTISPECTRAL: # this_image = pickle_loader(os.path.join(tissue_folder, patch_filename)) # read image, shape (H, W, 21) # mean = np.array(MEAN_CHANNEL_PIXELVALS) # std = np.array(STD_CHANNEL_PIXELVALS) # inp = (this_image - mean)/std # else: # this_image = mpimg.imread(os.path.join(tissue_folder, patch_filename)) # read image, shape (H, W, 3) # mean = np.array(MEAN_CHANNEL_PIXELVALS[:3]) # std = np.array(STD_CHANNEL_PIXELVALS[:3]) # inp = (this_image - mean)/std # ## plt.figure(), plt.imshow(this_image[:,:,:3]) ## print(os.path.join(tissue_folder, patch_filename)) ## sys.exit() # patches_session[j] = inp.transpose((2, 0, 1)) # # # Predict on patches # with torch.no_grad(): # inputs = torch.tensor(patches_session, dtype=torch.float).to(device) # outputs = model_ft(inputs) # _, preds = torch.max(outputs, 1) # ## print(preds) # # # Calculate number correct patches # true_label = tt # num_correctpatches_tissue_date[i] = np.sum(preds.cpu().numpy()==true_label) # iscorrect_tissue_date[i] = (num_correctpatches_tissue_date[i]/num_patches_tissue_date[i])>=0.5 # Assume 50% or greater patches predictions gives the overall specimen prediction # ## num_patches = num_patches + num_patches_tissue_date[i] ## print(' correct', num_correctpatches_tissue_date[i], num_patches_tissue_date[i], iscorrect_tissue_date[i]) ## print(num_patches) # # # Output per-specimen results # specimens_correct = np.sum(iscorrect_tissue_date) # print(' ', tissue, ': correct specimens ', specimens_correct, ' out of ', num_dates) else: # Train model from scratch print('Train model...') # Train #It should take around 15-25 min on CPU. On GPU though, it takes less than a minute. model_ft, cache_loss, cache_acc = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, dataloaders, device, dataset_sizes, num_epochs=NUM_EPOCHS) # Save loss and acc to disk filename_pre = 'nclass' + str(num_classes) t_size, val_acc = zip(*cache_acc['val']) # Calculate best val acc bestval_acc = max(val_acc).item() # filename_pre = timestr + '_nclass' + str(num_classes) + '_pretrain' + str(ISPRETRAINED)+ '_batch' + str(BATCH_SIZE) + '_epoch' + str(NUM_EPOCHS) + '_lr' + str(LEARNING_RATE) + '_' + str(LRDECAY_STEP) + '_' + str(LRDECAY_GAMMA) + '_val' +"{:.4f}".format(bestval_acc) filename_pre = timestr + '_multispec' + str(ISMULTISPECTRAL) + '_nclass' + str(num_classes) + '_pretrain' + str(ISPRETRAINED)+ '_batch' + str(BATCH_SIZE) + '_epoch' + str(NUM_EPOCHS) + '_lr' + str(LEARNING_RATE) + '_L2reg' + str(ALPHA_L2REG) + '_DROPOUT' + str(DROPOUT_RATE) + '_val' +"{:.4f}".format(bestval_acc) filename = 'lossacc_' + filename_pre + '.pkl' pickle.dump([cache_loss, cache_acc], open(os.path.join(out_path, filename), "wb" )) # Save trained model's parameters for inference filename2 = 'modelparam_' + filename_pre + '.pt' torch.save(model_ft.state_dict(), os.path.join(out_path, filename2)) # Evaluate model_ft.eval() # set dropout and batch normalization layers to evaluation mode before running inference # # Examine each figure and output to get granular prediction output info ## fig0 = visualize_model(model_ft, dataloaders, device, class_names, num_images=90, columns=10) ## fig0 = visualize_model(model_ft, dataloaders, device, class_names, num_images=10) # visualize validation images # fig0 = visualize_model(model_ft, dataloaders, device, class_names, num_images=288, columns=12, phase='test') # # Save visualization figure # fig0_filename = 'visualize_' + filename_pre + '.png' # fig0 = plt.gcf() # fig0.set_size_inches(FIG_HEIGHT, FIG_WIDTH) # plt.savefig(os.path.join(out_path, fig0_filename), bbox_inches='tight', dpi=FIG_DPI) fig1, fig2 = learning_curve(cache_loss, cache_acc, class_names, num_epochs=NUM_EPOCHS) # Save learning curve figures fig1_filename = 'losscurve_' + filename_pre + '.pdf' fig2_filename = 'acccurve_' + filename_pre + '.pdf' fig1.set_size_inches(FIG_HEIGHT, FIG_WIDTH) fig2.set_size_inches(FIG_HEIGHT, FIG_WIDTH) fig1.savefig(os.path.join(out_path, fig1_filename), bbox_inches='tight', dpi=FIG_DPI) fig2.savefig(os.path.join(out_path, fig2_filename), bbox_inches='tight', dpi=FIG_DPI) # Display confusion matrix for phase in ['train', 'val', 'test']: confusion_matrix = torch.zeros(num_classes, num_classes) y_actu = [] y_pred = [] with torch.no_grad(): for i, (inputs, classes) in enumerate(dataloaders[phase]): inputs = inputs.to(device, dtype=torch.float) # shape [128, 21, 36, 36] classes = classes.to(device) outputs = model_ft(inputs) _, preds = torch.max(outputs, 1) for t, p in zip(classes.view(-1), preds.view(-1)): confusion_matrix[t.long(), p.long()] += 1 # Vector of class labels and predictions y_actu = np.hstack((y_actu, classes.view(-1).cpu().numpy())) y_pred = np.hstack((y_pred, preds.view(-1).cpu().numpy())) # print(confusion_matrix) print(confusion_matrix.diag()/confusion_matrix.sum(1)) # per-class accuracy fig3 = plot_confusion_matrix(confusion_matrix, classes=class_names, normalize=CM_NORMALIZED, title='Confusion matrix, ' + phase) # Save confusion matrix figure fig3_filename = 'cm' + phase + '_' + filename_pre + '.pdf' fig3.set_size_inches(FIG_HEIGHT, FIG_WIDTH) fig3.savefig(os.path.join(out_path, fig3_filename), bbox_inches='tight', dpi=FIG_DPI) # Also save as jpg, 5-1-2020 fig3_filename_jpg = 'cm' + phase + '_' + filename_pre + '.jpg' fig3.savefig(os.path.join(out_path, fig3_filename_jpg), bbox_inches='tight', dpi=FIG_DPI) # Display confusion matrix analysis cm2 = pycm.ConfusionMatrix(actual_vector=y_actu, predict_vector=y_pred) # Create CM From Data # cm2 = pycm.ConfusionMatrix(matrix={"Class1": {"Class1": 1, "Class2":2}, "Class2": {"Class1": 0, "Class2": 5}}) # Create CM Directly cm2 # line output: pycm.ConfusionMatrix(classes: ['Class1', 'Class2']) print(cm2) plt.ioff() plt.show()
xmin=0, xmax=NUM_EPOCHS, colors=(0, 1, 0), linestyles='dashed' ) # random-guess loss marked as horizontal green dashed line plt.title('Learning curve, classes=' + str(class_names)) plt.xlabel('Epoch #') plt.ylabel('Accuracy') plt.legend(['train', 'val', 'random guess']) plt.show() #%%Finetuning the convnet #Load a pretrained model and reset final fully connected layer. #model_ft = models.resnet18(pretrained=True) model_ft = densenet_av.densenet_40_12_bc(pretrained=True) num_ftrs = model_ft.fc.in_features #model_ft.fc = nn.Linear(num_ftrs, 2) model_ft.fc = nn.Linear(num_ftrs, num_classes) model_ft = model_ft.to(device) criterion = nn.CrossEntropyLoss() # Observe that all parameters are being optimized #optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9) optimizer_ft = optim.Adam(model_ft.parameters(), lr=0.0001) # defaulat ADAM lr = 0.001 # Decay LR by a factor of 0.1 every 7 epochs exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)