def evaluate_bulk(eval_bulk):
data_dir = eval_bulk['data_dir']
prediction_path = eval_bulk['save_predictions_dir']
volumes_txt_file = eval_bulk['volumes_txt_file']
device = eval_bulk['device']
label_names = [
"vol_ID", "Background", "Left WM", "Left Cortex",
"Left Lateral ventricle", "Left Inf LatVentricle",
"Left Cerebellum WM", "Left Cerebellum Cortex", "Left Thalamus",
"Left Caudate", "Left Putamen", "Left Pallidum", "3rd Ventricle",
"4th Ventricle", "Brain Stem", "Left Hippocampus", "Left Amygdala",
"CSF (Cranial)", "Left Accumbens", "Left Ventral DC", "Right WM",
"Right Cortex", "Right Lateral Ventricle", "Right Inf LatVentricle",
"Right Cerebellum WM", "Right Cerebellum Cortex", "Right Thalamus",
"Right Caudate", "Right Putamen", "Right Pallidum",
"Right Hippocampus", "Right Amygdala", "Right Accumbens",
"Right Ventral DC"
]
batch_size = eval_bulk['batch_size']
need_unc = eval_bulk['estimate_uncertainty']
mc_samples = eval_bulk['mc_samples']
dir_struct = eval_bulk['directory_struct']
if eval_bulk['view_agg'] == 'True':
coronal_model_path = eval_bulk['coronal_model_path']
axial_model_path = eval_bulk['axial_model_path']
eu.evaluate2view(coronal_model_path, axial_model_path,
volumes_txt_file, data_dir, device, prediction_path,
batch_size, label_names, dir_struct, need_unc,
mc_samples)
else:
coronal_model_path = eval_bulk['coronal_model_path']
eu.evaluate(coronal_model_path, volumes_txt_file, data_dir, device,
prediction_path, batch_size, "COR", label_names,
dir_struct, need_unc, mc_samples)
def eval(self, model, data_iterators, key='val'):
assert key in ('val', 'test')
assert not (data_iterators[key] is None)
criterion = self.criterion
weight = self.weight
device = self.device
return evaluator.evaluate(model, device, data_iterators[key],
self.target_labels, criterion, weight)
def run_train(opt, training_data_loader, validation_data_loader):
if not os.path.exists(opt.checkpoint_dir):
os.makedirs(opt.checkpoint_dir)
log_file = os.path.join(opt.checkpoint_dir, 'vgg_log.csv')
print('[Initialize networks for training]')
net = VGG(opt)
L2_criterion = nn.MSELoss()
print(net)
if opt.resume:
opt.start_epoch, net = load_model(opt, opt.checkpoint_dir)
else:
with open(log_file, mode='w') as f:
f.write('epoch, train_loss, train_acc, valid_loss, valid_acc\n')
print('===> Setting GPU')
print('CUDA Available', torch.cuda.is_available())
if opt.use_cuda and torch.cuda.is_available():
opt.use_cuda = True
opt.device = 'cuda'
else:
opt.use_cuda = False
opt.device = 'cpu'
if torch.cuda.device_count() > 1 and opt.multi_gpu:
print("Use" + str(torch.cuda.device_count()) + 'GPUs')
net = nn.DataParallel(net)
if opt.use_cuda:
net = net.to(opt.device)
L2_criterion = L2_criterion.to(opt.device)
print("===> Setting Optimizer")
optimizer = torch.optim.Adam(net.parameters(),
lr=opt.lr,
betas=(opt.b1, opt.b2))
for epoch in range(opt.start_epoch, opt.n_epochs):
opt.epoch_num = epoch
train_loss, train_acc = train(opt,
net,
optimizer,
training_data_loader,
loss_criterion=L2_criterion)
valid_loss, valid_acc = evaluate(opt,
net,
validation_data_loader,
loss_criterion=L2_criterion)
with open(log_file, mode='a') as f:
f.write("%d, %08f,%08f,%08f,%08f\n" %
(epoch, train_loss, train_acc, valid_loss, valid_acc))
save_checkpoint(opt, net, epoch, valid_loss)
if epoch == 25:
optimizer = optim.Adam(model.module.parameters(), lr = 0.00005 )
h_params.teacher_forcing = 0.99
train_loss, train_cer = train(model, train_batch_num,
train_queue, criterion,
optimizer, device,
train_begin, h_params.workers,
10, h_params.teacher_forcing)
logger.info('Epoch %d (Training) Loss %0.4f CER %0.4f' % (epoch, train_loss, train_cer))
train_loader.join()
valid_queue = queue.Queue(h_params.workers * 2)
valid_loader = BaseDataLoader(valid_dataset, valid_queue, h_params.batch_size, 0)
valid_loader.start()
eval_loss, eval_cer = evaluate(model, valid_loader, valid_queue, criterion, device)
logger.info('Epoch %d (Evaluate) Loss %0.4f CER %0.4f' % (epoch, eval_loss, eval_cer))
valid_loader.join()
is_best_loss = (eval_loss < best_loss)
is_best_cer = (eval_cer < best_cer)
if is_best_loss:
torch.save(model, "./best_loss")
if is_best_cer:
torch.save(model, "./best_cer")
def evaluate_bulk(eval_bulk):
data_dir = eval_bulk['data_dir']
prediction_path = eval_bulk['save_predictions_dir']
volumes_txt_file = eval_bulk['volumes_txt_file']
device = eval_bulk['device']
#BORIS
# label_names = ["vol_ID", "Background", "Left WM", "Left Cortex", "Left Lateral ventricle", "Left Inf LatVentricle",
# "Left Cerebellum WM", "Left Cerebellum Cortex", "Left Thalamus", "Left Caudate", "Left Putamen",
# "Left Pallidum", "3rd Ventricle", "4th Ventricle", "Brain Stem", "Left Hippocampus", "Left Amygdala",
# "CSF (Cranial)", "Left Accumbens", "Left Ventral DC", "Right WM", "Right Cortex",
# "Right Lateral Ventricle", "Right Inf LatVentricle", "Right Cerebellum WM",
# "Right Cerebellum Cortex", "Right Thalamus", "Right Caudate", "Right Putamen", "Right Pallidum",
# "Right Hippocampus", "Right Amygdala", "Right Accumbens", "Right Ventral DC"]
label_names = [
"background", "3rd-Ventricle", "4th-Ventricle", "Right-Accumbens-Area",
"Left-Accumbens-Area", "Right-Amygdala", "Left-Amygdala", "Brain-Stem",
"Right-Caudate", "Left-Caudate", "Right-Cerebellum-Exterior",
"Left-Cerebellum-Exterior", "Right-Cerebellum-White-Matter",
"Left-Cerebellum-White-Matter", "Right-Cerebral-White-Matter",
"Left-Cerebral-White-Matter", "Right-Hippocampus", "Left-Hippocampus",
"Right-Inf-Lat-Vent", "Left-Inf-Lat-Vent", "Right-Lateral-Ventricle",
"Left-Lateral-Ventricle", "Right-Pallidum", "Left-Pallidum",
"Right-Putamen", "Left-Putamen", "Right-Thalamus-Proper",
"Left-Thalamus-Proper", "Right-Ventral-DC", "Left-Ventral-DC",
"Cerebellar-Vermal-Lobules-I-V", "Cerebellar-Vermal-Lobules-VI-VII",
"Cerebellar-Vermal-Lobules-VIII-X", "Left-Basal-Forebrain",
"Right-Basal-Forebrain", "Right-ACgG--anterior-cingulate-gyrus",
"Left-ACgG--anterior-cingulate-gyrus", "Right-AIns--anterior-insula",
"Left-AIns--anterior-insula", "Right-AOrG--anterior-orbital-gyrus",
"Left-AOrG--anterior-orbital-gyrus", "Right-AnG---angular-gyrus",
"Left-AnG---angular-gyrus", "Right-Calc--calcarine-cortex",
"Left-Calc--calcarine-cortex", "Right-CO----central-operculum",
"Left-CO----central-operculum", "Right-Cun---cuneus",
"Left-Cun---cuneus", "Right-Ent---entorhinal-area",
"Left-Ent---entorhinal-area", "Right-FO----frontal-operculum",
"Left-FO----frontal-operculum", "Right-FRP---frontal-pole",
"Left-FRP---frontal-pole", "Right-FuG---fusiform-gyrus",
"Left-FuG---fusiform-gyrus", "Right-GRe---gyrus-rectus",
"Left-GRe---gyrus-rectus", "Right-IOG---inferior-occipital-gyrus",
"Left-IOG---inferior-occipital-gyrus",
"Right-ITG---inferior-temporal-gyrus",
"Left-ITG---inferior-temporal-gyrus", "Right-LiG---lingual-gyrus",
"Left-LiG---lingual-gyrus", "Right-LOrG--lateral-orbital-gyrus",
"Left-LOrG--lateral-orbital-gyrus",
"Right-MCgG--middle-cingulate-gyrus",
"Left-MCgG--middle-cingulate-gyrus",
"Right-MFC---medial-frontal-cortex",
"Left-MFC---medial-frontal-cortex", "Right-MFG---middle-frontal-gyrus",
"Left-MFG---middle-frontal-gyrus",
"Right-MOG---middle-occipital-gyrus",
"Left-MOG---middle-occipital-gyrus",
"Right-MOrG--medial-orbital-gyrus", "Left-MOrG--medial-orbital-gyrus",
"Right-MPoG--postcentral-gyrus", "Left-MPoG--postcentral-gyrus",
"Right-MPrG--precentral-gyrus", "Left-MPrG--precentral-gyrus",
"Right-MSFG--superior-frontal-gyrus",
"Left-MSFG--superior-frontal-gyrus",
"Right-MTG---middle-temporal-gyrus",
"Left-MTG---middle-temporal-gyrus", "Right-OCP---occipital-pole",
"Left-OCP---occipital-pole", "Right-OFuG--occipital-fusiform-gyrus",
"Left-OFuG--occipital-fusiform-gyrus",
"Right-OpIFG-opercular-part-of-the-IFG",
"Left-OpIFG-opercular-part-of-the-IFG",
"Right-OrIFG-orbital-part-of-the-IFG",
"Left-OrIFG-orbital-part-of-the-IFG",
"Right-PCgG--posterior-cingulate-gyrus",
"Left-PCgG--posterior-cingulate-gyrus", "Right-PCu---precuneus",
"Left-PCu---precuneus", "Right-PHG---parahippocampal-gyrus",
"Left-PHG---parahippocampal-gyrus", "Right-PIns--posterior-insula",
"Left-PIns--posterior-insula", "Right-PO----parietal-operculum",
"Left-PO----parietal-operculum", "Right-PoG---postcentral-gyrus",
"Left-PoG---postcentral-gyrus", "Right-POrG--posterior-orbital-gyrus",
"Left-POrG--posterior-orbital-gyrus", "Right-PP----planum-polare",
"Left-PP----planum-polare", "Right-PrG---precentral-gyrus",
"Left-PrG---precentral-gyrus", "Right-PT----planum-temporale",
"Left-PT----planum-temporale", "Right-SCA---subcallosal-area",
"Left-SCA---subcallosal-area", "Right-SFG---superior-frontal-gyrus",
"Left-SFG---superior-frontal-gyrus",
"Right-SMC---supplementary-motor-cortex",
"Left-SMC---supplementary-motor-cortex",
"Right-SMG---supramarginal-gyrus", "Left-SMG---supramarginal-gyrus",
"Right-SOG---superior-occipital-gyrus",
"Left-SOG---superior-occipital-gyrus",
"Right-SPL---superior-parietal-lobule",
"Left-SPL---superior-parietal-lobule",
"Right-STG---superior-temporal-gyrus",
"Left-STG---superior-temporal-gyrus", "Right-TMP---temporal-pole",
"Left-TMP---temporal-pole", "Right-TrIFG-triangular-part-of-the-IFG",
"Left-TrIFG-triangular-part-of-the-IFG",
"Right-TTG---transverse-temporal-gyrus",
"Left-TTG---transverse-temporal-gyrus"
]
batch_size = eval_bulk['batch_size']
need_unc = eval_bulk['estimate_uncertainty']
mc_samples = eval_bulk['mc_samples']
dir_struct = eval_bulk['directory_struct']
if 'exit_on_error' in eval_bulk.keys():
exit_on_error = eval_bulk['exit_on_error']
else:
exit_on_error = False
if eval_bulk['view_agg'] == 'True':
coronal_model_path = eval_bulk['coronal_model_path']
axial_model_path = eval_bulk['axial_model_path']
eu.evaluate2view(coronal_model_path,
axial_model_path,
volumes_txt_file,
data_dir,
device,
prediction_path,
batch_size,
label_names,
dir_struct,
need_unc,
mc_samples,
exit_on_error=exit_on_error)
else:
coronal_model_path = eval_bulk['coronal_model_path']
eu.evaluate(coronal_model_path,
volumes_txt_file,
data_dir,
device,
prediction_path,
batch_size,
"COR",
label_names,
dir_struct,
need_unc,
mc_samples,
exit_on_error=exit_on_error)
def notebook_func(truth_path='val.json', ans_path='ans.json'):
truth_annotations, ans_annotations = json_to_pd(truth_path, ans_path)
return evaluate(truth_annotations, ans_annotations)
def main():
args = pars_args(sys.argv[1:])
truth_annotations, ans_annotations = json_to_pd(args.truth_path,
args.ans_path)
recall, precision = evaluate(truth_annotations, ans_annotations)
print(calc_MAP(precision, recall))