def test(FLG):
device = torch.device('cuda:{}'.format(FLG.devices[0]))
torch.set_grad_enabled(False)
torch.backends.cudnn.benchmark = True
torch.cuda.set_device(FLG.devices[0])
report = [ScoreReport() for _ in range(FLG.fold)]
overall_report = ScoreReport()
target_dict = np.load(pjoin(FLG.data_root, 'target_dict.pkl'))
with open(FLG.model + '_stat.pkl', 'rb') as f:
stat = pickle.load(f)
summary = Summary(port=10001, env=str(FLG.model) + 'CAM')
class Feature(object):
def __init__(self):
self.blob = None
def capture(self, blob):
self.blob = blob
if 'plane' in FLG.model:
model = Plane(len(FLG.labels), name=FLG.model)
elif 'resnet11' in FLG.model:
model = resnet11(len(FLG.labels), FLG.model)
elif 'resnet19' in FLG.model:
model = resnet19(len(FLG.labels), FLG.model)
elif 'resnet35' in FLG.model:
model = resnet35(len(FLG.labels), FLG.model)
elif 'resnet51' in FLG.model:
model = resnet51(len(FLG.labels), FLG.model)
else:
raise NotImplementedError(FLG.model)
model.to(device)
ad_h = []
nl_h = []
adcams = np.zeros((4, 3, 112, 144, 112), dtype="f8")
nlcams = np.zeros((4, 3, 112, 144, 112), dtype="f8")
sb = [9.996e-01, 6.3e-01, 1.001e-01]
for running_fold in range(FLG.fold):
_, validblock, _ = fold_split(
FLG.fold, running_fold, FLG.labels,
np.load(pjoin(FLG.data_root, 'subject_indices.npy')), target_dict)
validset = ADNIDataset(FLG.labels,
pjoin(FLG.data_root, FLG.modal),
validblock,
target_dict,
transform=transform_presets(FLG.augmentation))
validloader = DataLoader(validset, pin_memory=True)
epoch, _ = load_checkpoint(model, FLG.checkpoint_root, running_fold,
FLG.model, None, True)
model.eval()
feature = Feature()
def hook(mod, inp, oup):
return feature.capture(oup.data.cpu().numpy())
_ = model.layer4.register_forward_hook(hook)
fc_weights = model.fc.weight.data.cpu().numpy()
transformer = Compose([CenterCrop((112, 144, 112)), ToFloatTensor()])
im, _ = original_load(validblock, target_dict, transformer, device)
for image, target in validloader:
true = target
npatches = 1
if len(image.shape) == 6:
_, npatches, c, x, y, z = image.shape
image = image.view(-1, c, x, y, z)
target = torch.stack([target
for _ in range(npatches)]).squeeze()
image = image.cuda(device, non_blocking=True)
target = target.cuda(device, non_blocking=True)
output = model(image)
if npatches == 1:
score = F.softmax(output, dim=1)
else:
score = torch.mean(F.softmax(output, dim=1),
dim=0,
keepdim=True)
report[running_fold].update_true(true)
report[running_fold].update_score(score)
overall_report.update_true(true)
overall_report.update_score(score)
print(target)
if FLG.cam:
s = 0
cams = []
if target[0] == 0:
s = score[0][0]
#s = s.cpu().numpy()[()]
cams = adcams
else:
sn = score[0][1]
#s = s.cpu().numpy()[()]
cams = nlcams
if s > sb[0]:
cams[0] = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
cams[0],
s,
num_images=5)
elif s > sb[1]:
cams[1] = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
cams[1],
s,
num_images=5)
elif s > sb[2]:
cams[2] = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
cams[2],
s,
num_images=5)
else:
cams[3] = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
cams[3],
s,
num_images=5)
#ad_h += [s]
#nl_h += [sn]
print('At {}'.format(epoch))
print(
metrics.classification_report(report[running_fold].y_true,
report[running_fold].y_pred,
target_names=FLG.labels,
digits=4))
print('accuracy {}'.format(report[running_fold].accuracy))
#print(np.histogram(ad_h))
#print(np.histogram(nl_h))
print('over all')
print(
metrics.classification_report(overall_report.y_true,
overall_report.y_pred,
target_names=FLG.labels,
digits=4))
print('accuracy {}'.format(overall_report.accuracy))
with open(FLG.model + '_stat.pkl', 'wb') as f:
pickle.dump(report, f, pickle.HIGHEST_PROTOCOL)
def test_cam():
FLAGS.noadapt = True
summary = Summary(port=39199, env=FLAGS.visdom_env)
with open(FLAGS.model + '_stat.pkl', 'rb') as f:
stat = pickle.load(f)
class Feature(object):
def __init__(self):
self.blob = None
def capture(self, blob):
self.blob = blob
# TODO: cropped cam?
transformer = Compose([CenterCrop((112, 144, 112)), ToFloatTensor()])
model = create_model()
model = model.cuda(FLAGS.devices[0])
for running_k in range(FLAGS.kfold):
load_checkpoint(model,
FLAGS.checkpoint_root,
running_k,
FLAGS.model,
epoch=None,
is_best=True)
model.eval()
feature = Feature()
def hook(mod, inp, oup):
return feature.capture(oup.data.cpu().numpy())
_ = model.layer4.register_forward_hook(hook)
fc_weights = model.fc.weight.data.cpu().numpy()
_, validset, _ =\
make_kfold_dataset(FLAGS.kfold, running_k,
np.load(pjoin(FLAGS.root, 'sids.npy')),
np.load(pjoin(FLAGS.root, 'diagnosis.npz')),
FLAGS.labels, [FLAGS.dataset_root],
validset_loader=ScaledLoader(
(112, 144, 112), False),
valid_transform=transformer)
i, p = stat[running_k]['pos_best']['index'], stat[running_k][
'pos_best']['p']
i = 8
image, target = validset[i]
_ = model(Variable(image.view(1, *image.shape).cuda(FLAGS.devices[0])))
name = 'k' + str(running_k)
summary.cam3d(name + FLAGS.labels[target] + '_pos_best_' + str(p),
image,
feature.blob,
fc_weights,
target,
num_images=10)
i, p = stat[running_k]['neg_best']['index'], stat[running_k][
'neg_best']['p']
i = 45
image, target = validset[i]
_ = model(Variable(image.view(1, *image.shape).cuda(FLAGS.devices[0])))
summary.cam3d(name + FLAGS.labels[target] + '_neg_best_' + str(p),
image,
feature.blob,
fc_weights,
target,
num_images=10)
def test_cam():
with open(FLG.model + '_stat.pkl', 'rb') as f:
stat = pickle.load(f)
summary = Summary(port=10001, env=str(FLG.model) + 'CAM')
class Feature(object):
def __init__(self):
self.blob = None
def capture(self, blob):
self.blob = blob
# TODO: create model
device = torch.device('cuda:{}'.format(FLG.devices[0]))
torch.set_grad_enabled(False)
torch.backends.cudnn.benchmark = True
torch.cuda.set_device(FLG.devices[0])
report = [ScoreReport() for _ in range(FLG.fold)]
target_dict = np.load(pjoin(FLG.data_root, 'target_dict.pkl'))
model = Plane(len(FLG.labels), name=FLG.model)
model.to(device)
transformer = Compose([CenterCrop((112, 144, 112)), ToFloatTensor()])
def original_load(validblock):
originalset = ADNIDataset(FLG.labels,
pjoin(FLG.data_root, 'spm_normalized'),
validblock,
target_dict,
transform=transformer)
originloader = DataLoader(originalset, pin_memory=True)
for image, target in originloader:
if len(image.shape) == 6:
_, npatches, c, x, y, z = image.shape
image = image.view(-1, c, x, y, z)
target = torch.stack([target
for _ in range(npatches)]).squeeze()
image = image.cuda(device, non_blocking=True)
target = target.cuda(device, non_blocking=True)
break
return image, target
hadcams = np.zeros((3, 112, 144, 112), dtype="f8")
madcams = np.zeros((3, 112, 144, 112), dtype="f8")
sadcams = np.zeros((3, 112, 144, 112), dtype="f8")
zadcams = np.zeros((3, 112, 144, 112), dtype="f8")
nlcams = np.zeros((4, 3, 112, 144, 112), dtype="f8")
sb = [4.34444371e-16, 1.67179015e-18, 4.08813312e-23]
#im, _ = original_load(validblock)
for running_fold in range(FLG.fold):
# validset
_, validblock, _ = fold_split(
FLG.fold, running_fold, FLG.labels,
np.load(pjoin(FLG.data_root, 'subject_indices.npy')), target_dict)
validset = ADNIDataset(FLG.labels,
pjoin(FLG.data_root, FLG.modal),
validblock,
target_dict,
transform=transformer)
validloader = DataLoader(validset, pin_memory=True)
load_checkpoint(model,
FLG.checkpoint_root,
running_fold,
FLG.model,
epoch=None,
is_best=True)
model.eval()
feature = Feature()
def hook(mod, inp, oup):
return feature.capture(oup.data.cpu().numpy())
_ = model.layer4.register_forward_hook(hook)
fc_weights = model.fc.weight.data.cpu().numpy()
im, _ = original_load(validblock)
ad_s = []
for image, target in validloader:
true = target
npatches = 1
if len(image.shape) == 6:
_, npatches, c, x, y, z = image.shape
image = image.view(-1, c, x, y, z)
target = torch.stack([target
for _ in range(npatches)]).squeeze()
image = image.cuda(device, non_blocking=True)
target = target.cuda(device, non_blocking=True)
#_ = model(image.view(*image.shape))
output = model(image)
if npatches == 1:
score = F.softmax(output, dim=1)
else:
score = torch.mean(F.softmax(output, dim=1),
dim=0,
keepdim=True)
sa = score[0][1]
#name = 'k'+str(running_fold)
sa = sa.cpu().numpy()[()]
print(score, score.shape)
if true == torch.tensor([0]):
if sa > sb[0]:
hadcams = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
hadcams,
sa,
num_images=5)
elif sa > sb[1]:
madcams = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
madcams,
sa,
num_images=5)
elif sa > sb[2]:
sadcams = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
sadcams,
sa,
num_images=5)
else:
zadcams = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
zadcams,
sa,
num_images=5)
else:
if s > sb[0]:
nlcams[0] = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
nlcams[0],
sr,
num_images=5)
elif sr > sb[1]:
nlcams[1] = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
nlcams[1],
sr,
num_images=5)
elif sr > sb[2]:
nlcams[2] = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
nlcams[2],
sr,
num_images=5)
else:
nlcams[3] = summary.cam3d(FLG.labels[target],
im,
feature.blob,
fc_weights,
target,
nlcams[3],
sr,
num_images=5)
ad_s += [sr]
print('histogram', np.histogram(ad_s))