def load_data(PATH_TO_IMAGES, LABEL, PATH_TO_MODEL, POSITIVE_FINDINGS_ONLY,
STARTER_IMAGES):
"""
Loads dataloader and torchvision model
Args:
PATH_TO_IMAGES: path to NIH CXR images
LABEL: finding of interest (must exactly match one of FINDINGS defined below or will get error)
PATH_TO_MODEL: path to downloaded pretrained model or your own retrained model
POSITIVE_FINDINGS_ONLY: dataloader will show only examples + for LABEL pathology if True, otherwise shows positive
and negative examples if false
Returns:
dataloader: dataloader with test examples to show
model: fine tuned torchvision densenet-121
"""
checkpoint = torch.load(PATH_TO_MODEL,
map_location=lambda storage, loc: storage)
model = checkpoint['model']
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
del checkpoint
model.cpu()
# build dataloader on test
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
FINDINGS = [
'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration', 'Mass',
'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation', 'Edema',
'Emphysema', 'Fibrosis', 'Pleural_Thickening', 'Hernia'
]
data_transform = transforms.Compose([
transforms.Scale(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
if not POSITIVE_FINDINGS_ONLY:
finding = "any"
else:
finding = LABEL
dataset = CXR.CXRDataset(path_to_images=PATH_TO_IMAGES,
fold='test',
transform=data_transform,
finding=finding,
starter_images=STARTER_IMAGES)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=1)
return iter(dataloader), model
def train(self):
torch.manual_seed(0)
cudnn.benchmark = True
self.model.train()
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
data_transforms = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ColorJitter(brightness=0.1,
contrast=0.1,
saturation=0.1,
hue=0.1),
transforms.RandomAffine(degrees=10, scale=(.95, 1.05), shear=0),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
transformed_dataset = CXR.CXRDataset(path_to_images=self.data_path,
fold='train',
transform=data_transforms)
dataloader = torch.utils.data.DataLoader(transformed_dataset,
batch_size=16,
shuffle=True,
num_workers=8)
lr = 0.01
print(f"About to begin training on device: {self.device}")
for epoch in range(20):
for x, y, _ in dataloader:
x, y = Variable(x.to(self.device)), Variable(y.to(
self.device)).float()
output = self.model(x)
loss = self.loss_fn(output, y)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
lr *= 0.95
self.optimizer = get_optimizer(self.model, lr)
print(f"Completed epoch on {self.device}")
def load_data(label='any'):
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
PATH_TO_IMAGES = "dataset/images"
finding = 'any'
data_transform = transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
labels = [
'Atelectasis',
'Cardiomegaly',
'Effusion',
'Infiltration',
'Mass',
'Nodule',
'Pneumonia',
'Pneumothorax',
'Consolidation',
'Edema',
'Emphysema',
'Fibrosis',
'Pleural_Thickening',
'Hernia']
if label in labels:
finding = label
dataset = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='test',
transform=data_transform, finding=finding)
print('Total number of images is: {0}'.format(len(dataset.df)))
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=1)
return iter(dataloader), len(dataset.df)
def train_cnn(PATH_TO_IMAGES, LR, WEIGHT_DECAY):
"""
Train torchvision model to NIH data given high level hyperparameters.
Args:
PATH_TO_IMAGES: path to NIH images
LR: learning rate
WEIGHT_DECAY: weight decay parameter for SGD
Returns:
preds: torchvision model predictions on test fold with ground truth for comparison
aucs: AUCs for each train,test tuple
"""
NUM_EPOCHS = 100 #100
BATCH_SIZE = 32 #16
try:
rmtree('results/')
except BaseException:
pass # directory doesn't yet exist, no need to clear it
os.makedirs("results/")
# use imagenet mean,std for normalization
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
N_LABELS = 15 # we are predicting 15 labels. Originally 14 before adding Covid
# load labels
df = pd.read_csv("nih_labels_modified.csv", index_col=0)
# define torchvision transforms
data_transforms = {
'train':
transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.Resize(224), #was transforms.Scale
# because scale doesn't always give 224 x 224, this ensures 224 x
# 224
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'val':
transforms.Compose([
transforms.Resize(224), #was transforms.Scale
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
}
# create train/val dataloaders
transformed_datasets = {}
transformed_datasets['train'] = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='train',
transform=data_transforms['train'])
transformed_datasets['val'] = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='val',
transform=data_transforms['val'])
dataloaders = {}
dataloaders['train'] = torch.utils.data.DataLoader(
transformed_datasets['train'],
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=8)
dataloaders['val'] = torch.utils.data.DataLoader(
transformed_datasets['val'],
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=8)
# please do not attempt to train without GPU as will take excessively long
if not use_gpu:
raise ValueError("Error, requires GPU")
model = models.densenet121(pretrained=True)
num_ftrs = model.classifier.in_features
# add final layer with # outputs in same dimension of labels with sigmoid
# activation
model.classifier = nn.Sequential(nn.Linear(num_ftrs, N_LABELS),
nn.Sigmoid())
# put model on GPU
model = model.cuda()
# define criterion, optimizer for training
criterion = nn.BCELoss()
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=LR,
momentum=0.9,
weight_decay=WEIGHT_DECAY)
dataset_sizes = {x: len(transformed_datasets[x]) for x in ['train', 'val']}
# train model
model, best_epoch = train_model(model,
criterion,
optimizer,
LR,
num_epochs=NUM_EPOCHS,
dataloaders=dataloaders,
dataset_sizes=dataset_sizes,
weight_decay=WEIGHT_DECAY)
# get preds and AUCs on test fold
preds, aucs = E.make_pred_multilabel(data_transforms, model,
PATH_TO_IMAGES)
return preds, aucs
def make_pred_multilabel(data_transforms, model, PATH_TO_IMAGES):
"""
Gives predictions for test fold and calculates AUCs using previously trained model
Args:
data_transforms: torchvision transforms to preprocess raw images; same as validation transforms
model: densenet-121 from torchvision previously fine tuned to training data
PATH_TO_IMAGES: path at which NIH images can be found
Returns:
pred_df: dataframe containing individual predictions and ground truth for each test image
auc_df: dataframe containing aggregate AUCs by train/test tuples
"""
# calc preds in batches of 16, can reduce if your GPU has less RAM
BATCH_SIZE = 16
# set model to eval mode; required for proper predictions given use of batchnorm
model.train(False)
# create dataloader
dataset = CXR.CXRDataset(path_to_images=PATH_TO_IMAGES,
fold="test",
transform=data_transforms['val'])
dataloader = torch.utils.data.DataLoader(dataset,
BATCH_SIZE,
shuffle=False,
num_workers=8)
size = len(dataset)
# create empty dfs
pred_df = pd.DataFrame(columns=["Image Index"])
true_df = pd.DataFrame(columns=["Image Index"])
# iterate over dataloader
for i, data in enumerate(dataloader):
inputs, labels, _ = data
inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())
true_labels = labels.cpu().data.numpy()
batch_size = true_labels.shape
outputs = model(inputs)
probs = outputs.cpu().data.numpy()
# get predictions and true values for each item in batch
for j in range(0, batch_size[0]):
thisrow = {}
truerow = {}
thisrow["Image Index"] = dataset.df.index[BATCH_SIZE * i + j]
truerow["Image Index"] = dataset.df.index[BATCH_SIZE * i + j]
# iterate over each entry in prediction vector; each corresponds to
# individual label
for k in range(len(dataset.PRED_LABEL)):
thisrow["prob_" + dataset.PRED_LABEL[k]] = probs[j, k]
truerow[dataset.PRED_LABEL[k]] = true_labels[j, k]
pred_df = pred_df.append(thisrow, ignore_index=True)
true_df = true_df.append(truerow, ignore_index=True)
if (i % 10 == 0):
print(str(i * BATCH_SIZE))
auc_df = pd.DataFrame(columns=["label", "auc"])
# calc AUCs
for column in true_df:
if column not in [
'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration',
'Mass', 'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation',
'Edema', 'Emphysema', 'Fibrosis', 'Pleural_Thickening',
'Hernia'
]:
continue
actual = true_df[column]
pred = pred_df["prob_" + column]
thisrow = {}
thisrow['label'] = column
thisrow['auc'] = np.nan
try:
thisrow['auc'] = sklm.roc_auc_score(actual.as_matrix().astype(int),
pred.as_matrix())
except BaseException:
print("can't calculate auc for " + str(column))
auc_df = auc_df.append(thisrow, ignore_index=True)
pred_df.to_csv("results/preds.csv", index=False)
auc_df.to_csv("results/aucs.csv", index=False)
return pred_df, auc_df
def train_cnn(PATH_TO_IMAGES, LR, WEIGHT_DECAY, fine_tune=False, regression=False, freeze=False, adam=False,
initial_model_path=None, initial_brixia_model_path=None, weighted_cross_entropy_batchwise=False,
modification=None, weighted_cross_entropy=False):
"""
Train torchvision model to NIH data given high level hyperparameters.
Args:
PATH_TO_IMAGES: path to NIH images
LR: learning rate
WEIGHT_DECAY: weight decay parameter for SGD
Returns:
preds: torchvision model predictions on test fold with ground truth for comparison
aucs: AUCs for each train,test tuple
"""
NUM_EPOCHS = 100
BATCH_SIZE = 32
try:
rmtree('results/')
except BaseException:
pass # directory doesn't yet exist, no need to clear it
os.makedirs("results/")
# use imagenet mean,std for normalization
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
N_LABELS = 14 # we are predicting 14 labels
N_COVID_LABELS = 3 # we are predicting 3 COVID labels
# define torchvision transforms
data_transforms = {
'train': transforms.Compose([
# transforms.RandomHorizontalFlip(),
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'val': transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
}
# create train/val dataloaders
transformed_datasets = {}
transformed_datasets['train'] = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='train',
transform=data_transforms['train'],
fine_tune=fine_tune,
regression=regression)
transformed_datasets['val'] = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='val',
transform=data_transforms['val'],
fine_tune=fine_tune,
regression=regression)
dataloaders = {}
dataloaders['train'] = torch.utils.data.DataLoader(
transformed_datasets['train'],
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=8)
dataloaders['val'] = torch.utils.data.DataLoader(
transformed_datasets['val'],
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=8)
# please do not attempt to train without GPU as will take excessively long
if not use_gpu:
raise ValueError("Error, requires GPU")
if initial_model_path or initial_brixia_model_path:
if initial_model_path:
saved_model = torch.load(initial_model_path)
else:
saved_model = torch.load(initial_brixia_model_path)
model = saved_model['model']
del saved_model
if fine_tune and not initial_brixia_model_path:
num_ftrs = model.module.classifier.in_features
if freeze:
for feature in model.module.features:
for param in feature.parameters():
param.requires_grad = False
if feature == model.module.features.transition2:
break
if not regression:
model.module.classifier = nn.Linear(num_ftrs, N_COVID_LABELS)
else:
model.module.classifier = nn.Sequential(
nn.Linear(num_ftrs, 1),
nn.ReLU(inplace=True)
)
else:
model = models.densenet121(pretrained=True)
num_ftrs = model.classifier.in_features
model.classifier = nn.Linear(num_ftrs, N_LABELS)
if modification == 'transition_layer':
# num_ftrs = model.features.norm5.num_features
up1 = torch.nn.Sequential(torch.nn.ConvTranspose2d(num_ftrs, num_ftrs, kernel_size=3, stride=2, padding=1),
torch.nn.BatchNorm2d(num_ftrs),
torch.nn.ReLU(True))
up2 = torch.nn.Sequential(torch.nn.ConvTranspose2d(num_ftrs, num_ftrs, kernel_size=3, stride=2, padding=1),
torch.nn.BatchNorm2d(num_ftrs))
transition_layer = torch.nn.Sequential(up1, up2)
model.features.add_module('transition_chestX', transition_layer)
if modification == 'remove_last_block':
model.features.denseblock4 = nn.Sequential()
model.features.transition3 = nn.Sequential()
# model.features.norm5 = nn.BatchNorm2d(512)
# model.classifier = nn.Linear(512, N_LABELS)
if modification == 'remove_last_two_block':
model.features.denseblock4 = nn.Sequential()
model.features.transition3 = nn.Sequential()
model.features.transition2 = nn.Sequential()
model.features.denseblock3 = nn.Sequential()
model.features.norm5 = nn.BatchNorm2d(512)
model.classifier = nn.Linear(512, N_LABELS)
print(model)
# put model on GPU
if not initial_model_path:
model = nn.DataParallel(model)
model.to(device)
if regression:
criterion = nn.MSELoss()
else:
if weighted_cross_entropy:
pos_weights = transformed_datasets['train'].pos_neg_balance_weights()
print(pos_weights)
# pos_weights[pos_weights>40] = 40
criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weights)
else:
criterion = nn.BCEWithLogitsLoss()
if adam:
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=LR, weight_decay=WEIGHT_DECAY)
else:
optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=LR, weight_decay=WEIGHT_DECAY, momentum=0.9)
dataset_sizes = {x: len(transformed_datasets[x]) for x in ['train', 'val']}
# train model
if regression:
model, best_epoch = train_model(model, criterion, optimizer, LR, num_epochs=NUM_EPOCHS,
dataloaders=dataloaders, dataset_sizes=dataset_sizes,
weight_decay=WEIGHT_DECAY, fine_tune=fine_tune, regression=regression)
else:
model, best_epoch = train_model(model, criterion, optimizer, LR, num_epochs=NUM_EPOCHS,
dataloaders=dataloaders, dataset_sizes=dataset_sizes, weight_decay=WEIGHT_DECAY,
weighted_cross_entropy_batchwise=weighted_cross_entropy_batchwise,
fine_tune=fine_tune)
# get preds and AUCs on test fold
preds, aucs = E.make_pred_multilabel(dataloaders['val'], model, save_as_csv=False, fine_tune=fine_tune)
return preds, aucs
def run(PATH_TO_IMAGES, LR, WEIGHT_DECAY, opt):
"""
Train torchvision model to NIH data given high level hyperparameters.
Args:
PATH_TO_IMAGES: path to NIH images
LR: learning rate
WEIGHT_DECAY: weight decay parameter for SGD
Returns:
preds: torchvision model predictions on test fold with ground truth for comparison
aucs: AUCs for each train,test tuple
"""
use_gpu = torch.cuda.is_available()
gpu_count = torch.cuda.device_count()
print("Available GPU count:" + str(gpu_count))
wandb.init(project=opt.project, name=opt.run_name)
wandb.config.update(opt, allow_val_change=True)
NUM_EPOCHS = 60
BATCH_SIZE = opt.batch_size
if opt.eval_only:
# test only. it is okay to have duplicate run_path
os.makedirs(opt.run_path, exist_ok=True)
else:
# train from scratch, should not have the same run_path. Otherwise it will overwrite previous runs.
try:
os.makedirs(opt.run_path)
except FileExistsError:
print("[ERROR] run_path {} exists. try to assign a unique run_path".format(opt.run_path))
return None, None
except Exception as e:
print("exception while creating run_path {}".format(opt.run_path))
print(str(e))
return None, None
# use imagenet mean,std for normalization
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
N_LABELS = 14 # we are predicting 14 labels
# define torchvision transforms
if opt.random_crop:
data_transforms = {
'train': transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomResizedCrop(size=opt.input_size, scale=(0.8, 1.0)), # crop then resize
transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'val': transforms.Compose([
transforms.Resize(int(opt.input_size * 1.05)),
transforms.CenterCrop(opt.input_size),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
}
else:
data_transforms = {
'train': transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.Resize(opt.input_size),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'val': transforms.Compose([
transforms.Resize(opt.input_size),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
}
# create train/val dataloaders
transformed_datasets = {}
transformed_datasets['train'] = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='train',
transform=data_transforms['train'])
transformed_datasets['val'] = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='val',
transform=data_transforms['val'])
worker_init_fn = set_seed(opt)
dataloaders = {}
dataloaders['train'] = torch.utils.data.DataLoader(
transformed_datasets['train'],
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=30,
drop_last=True,
worker_init_fn=worker_init_fn
)
dataloaders['val'] = torch.utils.data.DataLoader(
transformed_datasets['val'],
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=30,
drop_last=True,
worker_init_fn=worker_init_fn
)
# please do not attempt to train without GPU as will take excessively long
if not use_gpu:
raise ValueError("Error, requires GPU")
# load model
model = load_model(N_LABELS, opt)
# define criterion, optimizer for training
criterion = nn.BCELoss()
optimizer = create_optimizer(model, LR, WEIGHT_DECAY, opt)
scheduler = lr_scheduler.ReduceLROnPlateau(
optimizer,
'max',
factor=opt.lr_decay_ratio,
patience=opt.patience,
verbose=True
)
dataset_sizes = {x: len(transformed_datasets[x]) for x in ['train', 'val']}
if opt.eval_only:
print("loading best model statedict")
# load best model weights to return
checkpoint_best = torch.load(os.path.join(opt.run_path, 'checkpoint'))
model = load_model(N_LABELS, opt=opt)
model.load_state_dict(checkpoint_best['state_dict'])
else:
# train model
model, best_epoch = train_model(
model,
criterion,
optimizer,
LR,
scheduler=scheduler,
num_epochs=NUM_EPOCHS,
dataloaders=dataloaders,
dataset_sizes=dataset_sizes,
PATH_TO_IMAGES=PATH_TO_IMAGES,
data_transforms=data_transforms,
opt=opt,
)
# get preds and AUCs on test fold
preds, aucs = E.make_pred_multilabel(
data_transforms,
model,
PATH_TO_IMAGES,
fold="test",
opt=opt,
)
wandb.log({
'val_official': np.average(list(aucs.auc))
})
return preds, aucs
def load_data(PATH_TO_IMAGES,
LABEL,
PATH_TO_MODEL,
fold,
POSITIVE_FINDINGS_ONLY=None,
covid=False):
"""
Loads dataloader and torchvision model
Args:
PATH_TO_IMAGES: path to NIH CXR images
LABEL: finding of interest (must exactly match one of FINDINGS defined below or will get error)
PATH_TO_MODEL: path to downloaded pretrained model or your own retrained model
POSITIVE_FINDINGS_ONLY: dataloader will show only examples + for LABEL pathology if True, otherwise shows positive
and negative examples if false
Returns:
dataloader: dataloader with test examples to show
model: fine tuned torchvision densenet-121
"""
checkpoint = torch.load(PATH_TO_MODEL,
map_location=lambda storage, loc: storage)
model = checkpoint['model']
del checkpoint
model = model.module.to(device)
# model.eval()
# for param in model.parameters():
# param.requires_grad_(False)
# build dataloader on test
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
data_transform = transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
if not covid:
bounding_box_transform = CXR.RescaleBB(224, 1024)
if not POSITIVE_FINDINGS_ONLY:
finding = "any"
else:
finding = LABEL
dataset = CXR.CXRDataset(path_to_images=PATH_TO_IMAGES,
fold=fold,
transform=data_transform,
transform_bb=bounding_box_transform,
finding=finding)
else:
dataset = CXR.CXRDataset(path_to_images=PATH_TO_IMAGES,
fold=fold,
transform=data_transform,
fine_tune=True)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=1)
return iter(dataloader), model
def samples_display(LABEL, beta=6):
dataset = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='BBox', #fold='train'
transform=data_transforms['train'],
transform_bb=bounding_box_transform,
fine_tune=False,
label_path=label_path,
finding=LABEL) #finding=LABEL
dataloader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=1)
global _label
_label = get_label(LABEL)
seed = np.random.randint(0, 1000000000)
np.random.seed(seed)
length = len(dataset)
print("20 Ransom samples of " + LABEL + " from " + str(length) + " images")
for sample_idx in np.random.choice(length,
20): #change with dataset length
iba = IBA(model.features.denseblock2)
iba.reset_estimate()
iba.estimate(model,
dataloader,
device=dev,
n_samples=length,
progbar=False) #change with dataset length
img, target, idx, bbox = dataset[sample_idx]
img = img[None].to(dev)
# reverse the data pre-processing for plotting the original image
np_img = tensor_to_np_img(img[0])
size = 4
rows = 1
cols = 5
fig, (ax, ax2, ax3, ax4,
ax5) = plt.subplots(1,
5,
figsize=(cols * (size + 1), rows * size))
cxr = img.data.cpu().numpy().squeeze().transpose(1, 2, 0)
mean = np.array([0.485, 0.456, 0.406])
std = np.array([0.229, 0.224, 0.225])
cxr = std * cxr + mean
cxr = np.clip(cxr, 0, 1)
# rect_original = patches.Rectangle((bbox[0, 0], bbox[0, 1]), bbox[0, 2], bbox[0, 3], linewidth=2, edgecolor='r', facecolor='none', zorder=2)
rect_original = patches.Rectangle((bbox[0], bbox[1]),
bbox[2],
bbox[3],
linewidth=2,
edgecolor='r',
facecolor='none',
zorder=2)
ax.imshow(cxr)
ax.axis('off')
ax.set_title(idx)
ax.add_patch(rect_original)
iba.reverse_lambda = False
iba.beta = beta
heatmap = iba.analyze(img, model_loss_closure)
# show the heatmap
ax2 = plot_saliency_map(heatmap, np_img, ax=ax2)
_ = ax2.set_title("Old method, Block2")
ax2.add_patch(
patches.Rectangle((bbox[0], bbox[1]),
bbox[2],
bbox[3],
linewidth=2,
edgecolor='r',
facecolor='none',
zorder=2))
iba.reverse_lambda = True
iba.beta = beta
heatmap = iba.analyze(img, model_loss_closure)
# show the heatmap
ax3 = plot_saliency_map(heatmap, np_img, ax=ax3)
_ = ax3.set_title("New method, Block2")
ax3.add_patch(
patches.Rectangle((bbox[0], bbox[1]),
bbox[2],
bbox[3],
linewidth=2,
edgecolor='r',
facecolor='none',
zorder=2))
iba = IBA(model.features.denseblock3)
iba.reset_estimate()
iba.estimate(model,
dataloader,
device=dev,
n_samples=length,
progbar=False)
iba.reverse_lambda = False
iba.beta = beta
heatmap = iba.analyze(img, model_loss_closure)
# show the heatmap
ax4 = plot_saliency_map(heatmap, np_img, ax=ax4)
_ = ax4.set_title("Old method, Block3")
ax4.add_patch(
patches.Rectangle((bbox[0], bbox[1]),
bbox[2],
bbox[3],
linewidth=2,
edgecolor='r',
facecolor='none',
zorder=2))
iba.reverse_lambda = True
iba.beta = beta
heatmap = iba.analyze(img, model_loss_closure)
# show the heatmap
ax5 = plot_saliency_map(heatmap, np_img, ax=ax5)
_ = ax5.set_title("New method, Block3")
ax5.add_patch(
patches.Rectangle((bbox[0], bbox[1]),
bbox[2],
bbox[3],
linewidth=2,
edgecolor='r',
facecolor='none',
zorder=2))
plt.show()
return length
zorder=2))
plt.show()
return length
"""# Cardiomegaly"""
LABEL = 'Cardiomegaly'
length_data = samples_display(LABEL)
dataset = CXR.CXRDataset(
path_to_images=PATH_TO_IMAGES,
fold='BBox', #fold='train'
transform=data_transforms['train'],
transform_bb=bounding_box_transform,
fine_tune=False,
label_path=label_path,
finding=LABEL) #finding=LABEL
dataloader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=1)
img, target, idx, bbox = dataset[0]
"""## Compare different betas (So I chose beta=6 instead of 10, the default)
###IB in denseblock3 with different betas
"""
iba = IBA(model.features.denseblock3)
iba.reset_estimate()
iba.estimate(model,
dataloader,
device=dev,
def make_pred_multilabel(model, PATH_TO_IMAGES, BATCH_SIZE=4):
"""
Gives predictions for test fold and calculates AUCs using previously trained model
Args:
data_transforms: torchvision transforms to preprocess raw images; same as validation transforms
model: the model to calculate the AUCs for
PATH_TO_IMAGES: path at which NIH images can be found
Returns:
pred_df: dataframe containing individual predictions and ground truth for each test image
auc_df: dataframe containing aggregate AUCs by train/test tuples
"""
# calc preds in batches of 16, can reduce if your GPU has less RAM
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
data_transforms = {
'train':
transforms.Compose([
transforms.RandomHorizontalFlip(),
# transforms.Resize(224),
# because scale doesn't always give 224 x 224, this ensures 224 x
# 224
# transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
'val':
transforms.Compose([
# transforms.Resize(224),
# transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean, std)
]),
}
# set model to eval mode; required for proper predictions given use of batchnorm
model.train(False)
# create dataloader
dataset = CXR.CXRDataset(path_to_images=PATH_TO_IMAGES,
fold="test",
sample=200,
transform=data_transforms['val'])
dataloader = torch.utils.data.DataLoader(dataset,
BATCH_SIZE,
shuffle=False,
num_workers=8)
size = len(dataset)
print(size)
# print('datasetsize', dataset_sizes)
print(dataset.df)
# create empty dfs
pred_df = pd.DataFrame(columns=["Image Index"])
true_df = pd.DataFrame(columns=["Image Index"])
# these lists will save values for the second way of calculating the scores
outputList = []
labelList = []
# iterate over dataloader
for idx, data in enumerate(dataloader):
inputs, labels, _ = data
inputs = Variable(inputs.to(device))
labels = Variable(labels.float().to(device))
true_labels = labels.cpu().data.numpy()
batch_size = true_labels.shape
outputs = model(inputs)
outputs = torch.sigmoid(outputs)
probs = outputs.cpu().data.numpy()
for i in range(outputs.shape[0]):
outputList.append(outputs[i].tolist())
labelList.append(labels[i].tolist())
# get predictions and true values for each item in batch
# here the dataframe 'true' and 'preds' are created by adding rows into them respectively
for j in range(0, batch_size[0]):
thisrow = {}
truerow = {}
thisrow["Image Index"] = dataset.df.index[BATCH_SIZE * idx + j]
truerow["Image Index"] = dataset.df.index[BATCH_SIZE * idx + j]
# iterate over each entry in prediction vector; each corresponds to
# individual label
for k in range(len(dataset.PRED_LABEL)):
thisrow["prob_" + dataset.PRED_LABEL[k]] = probs[j, k]
truerow[dataset.PRED_LABEL[k]] = true_labels[j, k]
pred_df = pred_df.append(thisrow, ignore_index=True)
true_df = true_df.append(truerow, ignore_index=True)
# print(pred_df)
# print(head(true_df))
if (idx % 100 == 0):
print(str(idx * BATCH_SIZE))
#Another way to calculate the AUCs. The calculation is done step by step.
print('Scores - Method2 -----------------------')
epoch_auc_ave = sklm.roc_auc_score(np.array(labelList),
np.array(outputList))
epoch_auc = sklm.roc_auc_score(np.array(labelList),
np.array(outputList),
average=None)
for i, c in enumerate(dataset.PRED_LABEL):
fpr, tpr, _ = sklm.roc_curve(
np.array(labelList)[:, i],
np.array(outputList)[:, i])
plt.plot(fpr, tpr, label=c)
print('{}: {:.4f} '.format(c, epoch_auc[i]))
print('Scores - Method2 -----------------------')
# here the auc scores are calculated and the 'auc' table is created
auc_df = pd.DataFrame(columns=["label", "auc"])
# calc AUCs
for column in true_df:
if column not in [
'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration',
'Mass', 'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation',
'Edema', 'Emphysema', 'Fibrosis', 'Pleural_Thickening',
'Hernia'
]:
continue
actual = true_df[column]
pred = pred_df["prob_" + column]
thisrow = {}
thisrow['label'] = column
thisrow['auc'] = np.nan
try:
thisrow['auc'] = sklm.roc_auc_score(actual.values.astype(int),
pred.values)
except BaseException as e:
print('-------------------')
print(e)
print(actual.values)
print(pred.values)
print('-------------------')
auc_df = auc_df.append(thisrow, ignore_index=True)
pred_df.to_csv("results/preds.csv", index=False)
auc_df.to_csv("results/aucs.csv", index=False)
true_df.to_csv('results/true.csv', index=False)
return pred_df, auc_df, true_df
