def load_arrinet_classifier(isprocessed=False, ismultispectral=True):
"""Classifier for tissue types of image patches
Dependencies: get_modelpath
"""
# Get model path and number of channels
modelpath, n_channels = get_modelpath(isprocessed=isprocessed, ismultispectral=ismultispectral)
print('Loading model path: ', modelpath)
# Initialize Arrinet
net = densenet_av.densenet_40_12_bc(in_channels=n_channels)
net.fc = nn.Linear(net.fc.in_features, NUM_CLASSES) # adjust last fully-connected layer dimensions
# net = net.to(device)
net.load_state_dict(torch.load(modelpath))
net.eval()
return net
def main():
# Get start time for this run
timestr = time.strftime("%Y%m%d-%H%M%S")
print(timestr)
# for tt in range(1):
for tt in range(30): # include this for loop to randomly sample learning rate, other hyperparameters
# Get start time for this run
timestr = time.strftime("%Y%m%d-%H%M%S")
print(timestr)
xx = 3 + random.random()*1
LEARNING_RATE = 10**-xx
# random search hyperparameters
yy = 3 + random.random()*1
ALPHA_L2REG = 1*10**-yy
zz = random.random()*0
DROPOUT_RATE = zz
print('Iteration: ', tt, LEARNING_RATE , ALPHA_L2REG, DROPOUT_RATE)
for xx in [True]:
# for xx in [True, False]:
ISMULTISPECTRAL = xx
#%% Data loading
if ISMULTISPECTRAL: # 21-channel pickled tiled images
# set paths to RGB images
data_dir = PATH_PATCHES3D
out_path = PATH_OUTPUT3D
# Data augmentation and normalization for training
# Just normalization for validation
data_transforms = {
'train': transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(MEAN_CHANNEL_PIXELVALS, STD_CHANNEL_PIXELVALS)
]),
'val': transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(MEAN_CHANNEL_PIXELVALS, STD_CHANNEL_PIXELVALS)
]),
'test': transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(MEAN_CHANNEL_PIXELVALS, STD_CHANNEL_PIXELVALS)
]),
}
num_channels = 21 - N_REMOVED
else: # RGB 3-channel, pickled images
# set paths to RGB images
data_dir = PATH_PATCHES2D
out_path = PATH_OUTPUT2D
# Data augmentation and normalization for training
# Just normalization for validation
data_transforms = {
'train': transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(MEAN_CHANNEL_PIXELVALS[:3], STD_CHANNEL_PIXELVALS[:3])
]),
'val': transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(MEAN_CHANNEL_PIXELVALS[:3], STD_CHANNEL_PIXELVALS[:3])
]),
'test': transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(MEAN_CHANNEL_PIXELVALS[:3], STD_CHANNEL_PIXELVALS[:3])
]),
}
num_channels = 3
image_datasets = {x: datasets.DatasetFolder(os.path.join(data_dir, x), loader=pickle_loader,
extensions='.pkl', transform=data_transforms[x])
for x in ['train', 'val', 'test']}
print('Num channels', num_channels)
if not LOADMODEL:
dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=BATCH_SIZE,
shuffle=True, num_workers=0)
for x in ['train', 'val', 'test']}
else:
dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=BATCH_SIZE,
shuffle=False, num_workers=0)
for x in ['train', 'val', 'test']}
dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val', 'test']}
class_names = image_datasets['train'].classes
num_classes = len(class_names)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('Dataset sizes', dataset_sizes)
print('Number of classes', num_classes)
print('GPU vs CPU:', device)
#%%Finetuning the convnet
#Load a pretrained model and reset final fully connected layer.
# model_ft = models.resnet18(pretrained=True)
model_ft = densenet_av.densenet_40_12_bc(pretrained=ISPRETRAINED, in_channels=num_channels, drop_rate=DROPOUT_RATE)
num_ftrs = model_ft.fc.in_features
print('model_ft.fc.in_features =', num_ftrs) #debugging
model_ft.fc = nn.Linear(num_ftrs, num_classes)
model_ft = model_ft.to(device)
criterion = nn.CrossEntropyLoss()
# Observe that all parameters are being optimized
# optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)
optimizer_ft = optim.Adam(model_ft.parameters(), lr=LEARNING_RATE, weight_decay=ALPHA_L2REG) # defaulat ADAM lr = 0.001
# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=LRDECAY_STEP, gamma=LRDECAY_GAMMA)
# exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=1000, gamma=LRDECAY_GAMMA) # For Adam optimizer, no need for LR decay
#%% Train and evaluate
if LOADMODEL: # load weights instead of training
print('Loading model... Select loss/acc file:')
filepath = mat.uigetfile()
[cache_loss, cache_acc] = pickle.load(open(filepath, "rb"))
print('Loading model... ')
modelpath = filepath.replace('lossacc', 'modelparam').replace('.pkl', '.pt')
model_ft.load_state_dict(torch.load(modelpath))
model_ft.eval()
# Get same filename for saving
path_head, path_tail = os.path.split(filepath)
filename_pre, path_ext = os.path.splitext(path_tail)
# # 8-25-2019
# # Obtain per-image classification accuracy based on patches - loop through folders without dataloader
# for phase in ['train', 'val', 'test']:
# data_dir2 = os.path.join(data_dir, phase)
# tissues = os.listdir(data_dir2) # should be num_classes # of folders
# print('Evaluating per-specimen accuracy on dataset: ', phase)
#
# # Iterate over tissue classes
# for tt, tissue in enumerate(tissues):
# tissue_folder = os.path.join(data_dir2, tissue)
# tissue_files = os.listdir(tissue_folder)
# tissue_dates = [i.split('_', 1)[0] for i in tissue_files]
# unique_dates = list(set(tissue_dates))
## print(unique_dates)
# num_dates = np.size(unique_dates)
#
# num_patches_tissue_date = np.zeros((num_dates, 1))
# num_correctpatches_tissue_date = np.zeros((num_dates, 1))
# iscorrect_tissue_date = np.zeros((num_dates, 1))
#
# # Calculate fraction of correct patch predictions per tissue-date specimen
# num_patches = 0
# for i, session in enumerate(unique_dates):
## print(session)
# num_patches_tissue_date[i] = tissue_dates.count(session)
# tissue_patches_session_filenames = [item for item in tissue_files if item.startswith(session)]
#
# # Load patches into one batch of shape [M, C, H, W]
# # where M is batch size (# patches), C is # channels
# patches_session = np.zeros((int(num_patches_tissue_date[i]), num_channels, TILE_HEIGHT, TILE_WIDTH))
# for j, patch_filename in enumerate(tissue_patches_session_filenames):
# if ISMULTISPECTRAL:
# this_image = pickle_loader(os.path.join(tissue_folder, patch_filename)) # read image, shape (H, W, 21)
# mean = np.array(MEAN_CHANNEL_PIXELVALS)
# std = np.array(STD_CHANNEL_PIXELVALS)
# inp = (this_image - mean)/std
# else:
# this_image = mpimg.imread(os.path.join(tissue_folder, patch_filename)) # read image, shape (H, W, 3)
# mean = np.array(MEAN_CHANNEL_PIXELVALS[:3])
# std = np.array(STD_CHANNEL_PIXELVALS[:3])
# inp = (this_image - mean)/std
#
## plt.figure(), plt.imshow(this_image[:,:,:3])
## print(os.path.join(tissue_folder, patch_filename))
## sys.exit()
# patches_session[j] = inp.transpose((2, 0, 1))
#
# # Predict on patches
# with torch.no_grad():
# inputs = torch.tensor(patches_session, dtype=torch.float).to(device)
# outputs = model_ft(inputs)
# _, preds = torch.max(outputs, 1)
#
## print(preds)
#
# # Calculate number correct patches
# true_label = tt
# num_correctpatches_tissue_date[i] = np.sum(preds.cpu().numpy()==true_label)
# iscorrect_tissue_date[i] = (num_correctpatches_tissue_date[i]/num_patches_tissue_date[i])>=0.5 # Assume 50% or greater patches predictions gives the overall specimen prediction
#
## num_patches = num_patches + num_patches_tissue_date[i]
## print(' correct', num_correctpatches_tissue_date[i], num_patches_tissue_date[i], iscorrect_tissue_date[i])
## print(num_patches)
#
# # Output per-specimen results
# specimens_correct = np.sum(iscorrect_tissue_date)
# print(' ', tissue, ': correct specimens ', specimens_correct, ' out of ', num_dates)
else: # Train model from scratch
print('Train model...')
# Train
#It should take around 15-25 min on CPU. On GPU though, it takes less than a minute.
model_ft, cache_loss, cache_acc = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
dataloaders, device, dataset_sizes, num_epochs=NUM_EPOCHS)
# Save loss and acc to disk
filename_pre = 'nclass' + str(num_classes)
t_size, val_acc = zip(*cache_acc['val']) # Calculate best val acc
bestval_acc = max(val_acc).item()
# filename_pre = timestr + '_nclass' + str(num_classes) + '_pretrain' + str(ISPRETRAINED)+ '_batch' + str(BATCH_SIZE) + '_epoch' + str(NUM_EPOCHS) + '_lr' + str(LEARNING_RATE) + '_' + str(LRDECAY_STEP) + '_' + str(LRDECAY_GAMMA) + '_val' +"{:.4f}".format(bestval_acc)
filename_pre = timestr + '_multispec' + str(ISMULTISPECTRAL) + '_nclass' + str(num_classes) + '_pretrain' + str(ISPRETRAINED)+ '_batch' + str(BATCH_SIZE) + '_epoch' + str(NUM_EPOCHS) + '_lr' + str(LEARNING_RATE) + '_L2reg' + str(ALPHA_L2REG) + '_DROPOUT' + str(DROPOUT_RATE) + '_val' +"{:.4f}".format(bestval_acc)
filename = 'lossacc_' + filename_pre + '.pkl'
pickle.dump([cache_loss, cache_acc], open(os.path.join(out_path, filename), "wb" ))
# Save trained model's parameters for inference
filename2 = 'modelparam_' + filename_pre + '.pt'
torch.save(model_ft.state_dict(), os.path.join(out_path, filename2))
# Evaluate
model_ft.eval() # set dropout and batch normalization layers to evaluation mode before running inference
# # Examine each figure and output to get granular prediction output info
## fig0 = visualize_model(model_ft, dataloaders, device, class_names, num_images=90, columns=10)
## fig0 = visualize_model(model_ft, dataloaders, device, class_names, num_images=10) # visualize validation images
# fig0 = visualize_model(model_ft, dataloaders, device, class_names, num_images=288, columns=12, phase='test')
# # Save visualization figure
# fig0_filename = 'visualize_' + filename_pre + '.png'
# fig0 = plt.gcf()
# fig0.set_size_inches(FIG_HEIGHT, FIG_WIDTH)
# plt.savefig(os.path.join(out_path, fig0_filename), bbox_inches='tight', dpi=FIG_DPI)
fig1, fig2 = learning_curve(cache_loss, cache_acc, class_names, num_epochs=NUM_EPOCHS)
# Save learning curve figures
fig1_filename = 'losscurve_' + filename_pre + '.pdf'
fig2_filename = 'acccurve_' + filename_pre + '.pdf'
fig1.set_size_inches(FIG_HEIGHT, FIG_WIDTH)
fig2.set_size_inches(FIG_HEIGHT, FIG_WIDTH)
fig1.savefig(os.path.join(out_path, fig1_filename), bbox_inches='tight', dpi=FIG_DPI)
fig2.savefig(os.path.join(out_path, fig2_filename), bbox_inches='tight', dpi=FIG_DPI)
# Display confusion matrix
for phase in ['train', 'val', 'test']:
confusion_matrix = torch.zeros(num_classes, num_classes)
y_actu = []
y_pred = []
with torch.no_grad():
for i, (inputs, classes) in enumerate(dataloaders[phase]):
inputs = inputs.to(device, dtype=torch.float) # shape [128, 21, 36, 36]
classes = classes.to(device)
outputs = model_ft(inputs)
_, preds = torch.max(outputs, 1)
for t, p in zip(classes.view(-1), preds.view(-1)):
confusion_matrix[t.long(), p.long()] += 1
# Vector of class labels and predictions
y_actu = np.hstack((y_actu, classes.view(-1).cpu().numpy()))
y_pred = np.hstack((y_pred, preds.view(-1).cpu().numpy()))
# print(confusion_matrix)
print(confusion_matrix.diag()/confusion_matrix.sum(1)) # per-class accuracy
fig3 = plot_confusion_matrix(confusion_matrix, classes=class_names, normalize=CM_NORMALIZED,
title='Confusion matrix, ' + phase)
# Save confusion matrix figure
fig3_filename = 'cm' + phase + '_' + filename_pre + '.pdf'
fig3.set_size_inches(FIG_HEIGHT, FIG_WIDTH)
fig3.savefig(os.path.join(out_path, fig3_filename), bbox_inches='tight', dpi=FIG_DPI)
# Also save as jpg, 5-1-2020
fig3_filename_jpg = 'cm' + phase + '_' + filename_pre + '.jpg'
fig3.savefig(os.path.join(out_path, fig3_filename_jpg), bbox_inches='tight', dpi=FIG_DPI)
# Display confusion matrix analysis
cm2 = pycm.ConfusionMatrix(actual_vector=y_actu, predict_vector=y_pred) # Create CM From Data
# cm2 = pycm.ConfusionMatrix(matrix={"Class1": {"Class1": 1, "Class2":2}, "Class2": {"Class1": 0, "Class2": 5}}) # Create CM Directly
cm2 # line output: pycm.ConfusionMatrix(classes: ['Class1', 'Class2'])
print(cm2)
plt.ioff()
plt.show()
xmin=0,
xmax=NUM_EPOCHS,
colors=(0, 1, 0),
linestyles='dashed'
) # random-guess loss marked as horizontal green dashed line
plt.title('Learning curve, classes=' + str(class_names))
plt.xlabel('Epoch #')
plt.ylabel('Accuracy')
plt.legend(['train', 'val', 'random guess'])
plt.show()
#%%Finetuning the convnet
#Load a pretrained model and reset final fully connected layer.
#model_ft = models.resnet18(pretrained=True)
model_ft = densenet_av.densenet_40_12_bc(pretrained=True)
num_ftrs = model_ft.fc.in_features
#model_ft.fc = nn.Linear(num_ftrs, 2)
model_ft.fc = nn.Linear(num_ftrs, num_classes)
model_ft = model_ft.to(device)
criterion = nn.CrossEntropyLoss()
# Observe that all parameters are being optimized
#optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)
optimizer_ft = optim.Adam(model_ft.parameters(),
lr=0.0001) # defaulat ADAM lr = 0.001
# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)