def test_cnn(output_dir,
data_loader,
subset_name,
split,
criterion,
cnn_index,
model_options,
gpu=False):
for selection in ["best_balanced_accuracy", "best_loss"]:
# load the best trained model during the training
model = create_model(model_options.model,
gpu,
dropout=model_options.dropout)
model, best_epoch = load_model(model,
os.path.join(output_dir,
'fold-%i' % split,
'models',
'cnn-%i' % cnn_index,
selection),
gpu=gpu,
filename='model_best.pth.tar')
results_df, metrics = test(model, data_loader, gpu, criterion,
model_options.mode)
print("%s level balanced accuracy is %f" %
(model_options.mode, metrics['balanced_accuracy']))
mode_level_to_tsvs(output_dir,
results_df,
metrics,
split,
selection,
model_options.mode,
dataset=subset_name,
cnn_index=cnn_index)
def test_cnn(data_loader, subset_name, split, criterion, options):
for selection in ["best_acc", "best_loss"]:
# load the best trained model during the training
model = init_model(options.model,
gpu=options.gpu,
dropout=options.dropout)
model, best_epoch = load_model(model,
os.path.join(options.output_dir,
'best_model_dir',
"fold_%i" % split, 'CNN',
selection),
gpu=options.gpu,
filename='model_best.pth.tar')
results_df, metrics = test(model, data_loader, options.gpu, criterion,
options.mode)
print("Slice level balanced accuracy is %f" %
metrics['balanced_accuracy'])
mode_level_to_tsvs(options.output_dir,
results_df,
metrics,
split,
selection,
options.mode,
dataset=subset_name)
def test_cnn(output_dir,
data_loader,
subset_name,
split,
criterion,
model_options,
gpu=False,
multiclass=False):
for selection in ["best_balanced_accuracy", "best_loss"]:
# load the best trained model during the training
model = create_model(model_options.model,
gpu,
dropout=model_options.dropout)
model, best_epoch = load_model(model,
os.path.join(output_dir,
'fold-%i' % split,
'models', selection),
gpu=gpu,
filename='model_best.pth.tar')
results_df, metrics = test(model, data_loader, gpu, criterion,
model_options.mode, multiclass)
print("%s level balanced accuracy is %f" %
(model_options.mode, metrics['balanced_accuracy']))
mode_level_to_tsvs(output_dir,
results_df,
metrics,
split,
selection,
model_options.mode,
dataset=subset_name)
# Soft voting
if model_options.mode in ["patch", "roi", "slice"]:
soft_voting_to_tsvs(
output_dir,
split,
selection=selection,
mode=model_options.mode,
dataset=subset_name,
selection_threshold=model_options.selection_threshold)
def inference_from_model_generic(caps_dir,
tsv_path,
model_path,
model_options,
num_cnn=None,
selection="best_balanced_accuracy"):
'''
Inference using an image/subject CNN model
'''
from os.path import join
gpu = not model_options.use_cpu
# Recreate the model with the network described in the json file
# Initialize the model
model = create_model(model_options.model,
gpu,
dropout=model_options.dropout)
transformations = get_transforms(model_options.mode,
model_options.minmaxnormalization)
# Define loss and optimizer
criterion = nn.CrossEntropyLoss()
if model_options.mode_task == 'multicnn':
predictions_df = pd.DataFrame()
metrics_df = pd.DataFrame()
for n in range(num_cnn):
dataset = return_dataset(model_options.mode,
caps_dir,
tsv_path,
model_options.preprocessing,
transformations,
model_options,
cnn_index=n)
test_loader = DataLoader(dataset,
batch_size=model_options.batch_size,
shuffle=False,
num_workers=model_options.nproc,
pin_memory=True)
# load the best trained model during the training
model, best_epoch = load_model(model,
join(model_path, 'cnn-%i' % n,
selection),
gpu,
filename='model_best.pth.tar')
cnn_df, cnn_metrics = test(model,
test_loader,
gpu,
criterion,
mode=model_options.mode)
predictions_df = pd.concat([predictions_df, cnn_df])
metrics_df = pd.concat(
[metrics_df, pd.DataFrame(cnn_metrics, index=[0])])
predictions_df.reset_index(drop=True, inplace=True)
metrics_df.reset_index(drop=True, inplace=True)
else:
# Load model from path
best_model, best_epoch = load_model(model,
join(model_path, selection),
gpu,
filename='model_best.pth.tar')
# Read/localize the data
data_to_test = return_dataset(model_options.mode, caps_dir, tsv_path,
model_options.preprocessing,
transformations, model_options)
# Load the data
test_loader = DataLoader(data_to_test,
batch_size=model_options.batch_size,
shuffle=False,
num_workers=model_options.nproc,
pin_memory=True)
# Run the model on the data
predictions_df, metrics = test(best_model,
test_loader,
gpu,
criterion,
mode=model_options.mode)
metrics_df = pd.DataFrame(metrics, index=[0])
return predictions_df, metrics_df
def test_cnn(output_dir, data_loader, subset_name, split, criterion, cnn_index, model_options, gpu=False, train_begin_time=None):
metric_dict = {}
for selection in ["best_balanced_accuracy", "best_loss"]:
# load the best trained model during the training
if model_options.model == 'UNet3D':
print('********** init UNet3D model for test! **********')
model = create_model(model_options.model, gpu=model_options.gpu, dropout=model_options.dropout, device_index=model_options.device, in_channels=model_options.in_channels,
out_channels=model_options.out_channels, f_maps=model_options.f_maps, layer_order=model_options.layer_order, num_groups=model_options.num_groups, num_levels=model_options.num_levels)
elif model_options.model == 'ResidualUNet3D':
print('********** init ResidualUNet3D model for test! **********')
model = create_model(model_options.model, gpu=model_options.gpu, dropout=model_options.dropout, device_index=model_options.device, in_channels=model_options.in_channels,
out_channels=model_options.out_channels, f_maps=model_options.f_maps, layer_order=model_options.layer_order, num_groups=model_options.num_groups, num_levels=model_options.num_levels)
elif model_options.model == 'UNet3D_add_more_fc':
print('********** init UNet3D_add_more_fc model for test! **********')
model = create_model(model_options.model, gpu=model_options.gpu, dropout=model_options.dropout, device_index=model_options.device, in_channels=model_options.in_channels,
out_channels=model_options.out_channels, f_maps=model_options.f_maps, layer_order=model_options.layer_order, num_groups=model_options.num_groups, num_levels=model_options.num_levels)
elif model_options.model == 'ResidualUNet3D_add_more_fc':
print('********** init ResidualUNet3D_add_more_fc model for test! **********')
model = create_model(model_options.model, gpu=model_options.gpu, dropout=model_options.dropout, device_index=model_options.device, in_channels=model_options.in_channels,
out_channels=model_options.out_channels, f_maps=model_options.f_maps, layer_order=model_options.layer_order, num_groups=model_options.num_groups, num_levels=model_options.num_levels)
elif model_options.model == 'VoxCNN':
print('********** init VoxCNN model for test! **********')
model = create_model(model_options.model, gpu=model_options.gpu, device_index=model_options.device)
elif model_options.model == 'ConvNet3D':
print('********** init ConvNet3D model for test! **********')
model = create_model(model_options.model, gpu=model_options.gpu, device_index=model_options.device)
elif 'gcn' in model_options.model:
print('********** init {}-{} model for test! **********'.format(model_options.model, model_options.gnn_type))
model = create_model(model_options.model, gpu=model_options.gpu, device_index=model_options.device, gnn_type=model_options.gnn_type,
gnn_dropout=model_options.gnn_dropout,
gnn_dropout_adj=model_options.gnn_dropout_adj,
gnn_non_linear=model_options.gnn_non_linear,
gnn_undirected=model_options.gnn_undirected,
gnn_self_loop=model_options.gnn_self_loop,
gnn_threshold = model_options.gnn_threshold,)
elif model_options.model == 'ROI_GCN':
print('********** init ROI_GCN model for test! **********')
model = create_model(model_options.model, gpu=model_options.gpu, device_index=model_options.device,
gnn_type=model_options.gnn_type,
gnn_dropout=model_options.gnn_dropout,
gnn_dropout_adj=model_options.gnn_dropout_adj,
gnn_non_linear=model_options.gnn_non_linear,
gnn_undirected=model_options.gnn_undirected,
gnn_self_loop=model_options.gnn_self_loop,
gnn_threshold = model_options.gnn_threshold,
nodel_vetor_layer=model_options.nodel_vetor_layer,
classify_layer=model_options.classify_layer,
num_node_features=model_options.num_node_features, num_class=model_options.num_class,
roi_size=model_options.roi_size, num_nodes=model_options.num_nodes,
gnn_pooling_layers=model_options.gnn_pooling_layers, global_sort_pool_k=model_options.global_sort_pool_k,
layers=model_options.layers,
shortcut_type=model_options.shortcut_type, use_nl=model_options.use_nl,
dropout=model_options.dropout,
device=model_options.device)
elif model_options.model == 'SwinTransformer3d':
print('********** init SwinTransformer3d model for test! **********')
model = create_model(model_options.model, gpu=model_options.gpu, dropout=model_options.dropout,
device_index=model_options.device,
sw_patch_size=model_options.sw_patch_size,
window_size = model_options.window_size,
mlp_ratio = model_options.mlp_ratio,
drop_rate = model_options.drop_rate,
attn_drop_rate = model_options.attn_drop_rate,
drop_path_rate = model_options.drop_path_rate,
qk_scale = model_options.qk_scale,
embed_dim = model_options.embed_dim,
depths = model_options.depths,
num_heads = model_options.num_heads,
qkv_bias = model_options.qkv_bias,
ape = model_options.ape,
patch_norm = model_options.patch_norm,
)
else:
model = create_model(model_options.model, gpu=model_options.gpu, dropout=model_options.dropout, device_index=model_options.device)
model, best_epoch = load_model(model, os.path.join(output_dir, 'fold-%i' % split, 'models',
'cnn-%i' % cnn_index, selection),
gpu=gpu, filename='model_best.pth.tar', device_index=model_options.device)
results_df, metrics = test(model, data_loader, gpu, criterion, model_options.mode, device_index=model_options.device, train_begin_time=train_begin_time)
print("[%s]: %s level balanced accuracy is %f" % (timeSince(train_begin_time), model_options.mode, metrics['balanced_accuracy']))
print('[{}]: {}_{}_result_df:'.format(timeSince(train_begin_time), subset_name, selection))
print(results_df)
print('[{}]: {}_{}_metrics:\n{}'.format(timeSince(train_begin_time), subset_name, selection, metrics))
wandb.log({'{}_accuracy_{}_singel_model'.format(subset_name, selection): metrics['accuracy'],
'{}_balanced_accuracy_{}_singel_model'.format(subset_name, selection): metrics['balanced_accuracy'],
'{}_sensitivity_{}_singel_model'.format(subset_name, selection): metrics['sensitivity'],
'{}_specificity_{}_singel_model'.format(subset_name, selection): metrics['specificity'],
'{}_ppv_{}_singel_model'.format(subset_name, selection): metrics['ppv'],
'{}_npv_{}_singel_model'.format(subset_name, selection): metrics['npv'],
'{}_total_loss_{}_singel_model'.format(subset_name, selection): metrics['total_loss'],
})
mode_level_to_tsvs(output_dir, results_df, metrics, split, selection, model_options.mode,
dataset=subset_name, cnn_index=cnn_index)
# return metric dict
metric_temp_dict = {'{}_accuracy_{}_singel_model'.format(subset_name, selection): metrics['accuracy'],
'{}_balanced_accuracy_{}_singel_model'.format(subset_name, selection): metrics['balanced_accuracy'],
'{}_sensitivity_{}_singel_model'.format(subset_name, selection): metrics['sensitivity'],
'{}_specificity_{}_singel_model'.format(subset_name, selection): metrics['specificity'],
'{}_ppv_{}_singel_model'.format(subset_name, selection): metrics['ppv'],
'{}_npv_{}_singel_model'.format(subset_name, selection): metrics['npv'],
'{}_total_loss_{}_singel_model'.format(subset_name, selection): metrics['total_loss'],
}
metric_dict.update(metric_temp_dict)
return metric_dict
def inference_from_model_generic(caps_dir,
tsv_path,
model_path,
model_options,
prefix,
output_dir,
fold,
selection,
labels=True,
num_cnn=None,
logger=None):
from os.path import join
import logging
if logger is None:
logger = logging
gpu = not model_options.use_cpu
_, all_transforms = get_transforms(model_options.mode,
model_options.minmaxnormalization)
test_df = load_data_test(tsv_path, model_options.diagnoses)
# Define loss and optimizer
criterion = get_criterion(model_options.loss)
if model_options.mode_task == 'multicnn':
for n in range(num_cnn):
test_dataset = return_dataset(model_options.mode,
caps_dir,
test_df,
model_options.preprocessing,
train_transformations=None,
all_transformations=all_transforms,
params=model_options,
cnn_index=n,
labels=labels)
test_loader = DataLoader(test_dataset,
batch_size=model_options.batch_size,
shuffle=False,
num_workers=model_options.nproc,
pin_memory=True)
# load the best trained model during the training
model = create_model(model_options, test_dataset.size)
model, best_epoch = load_model(model,
join(model_path, 'cnn-%i' % n,
selection),
gpu,
filename='model_best.pth.tar')
cnn_df, cnn_metrics = test(model,
test_loader,
gpu,
criterion,
mode=model_options.mode,
use_labels=labels)
if labels:
logger.info(
"%s balanced accuracy is %f for %s %i and model selected on %s"
% (prefix, cnn_metrics["balanced_accuracy"],
model_options.mode, n, selection))
mode_level_to_tsvs(output_dir,
cnn_df,
cnn_metrics,
fold,
selection,
model_options.mode,
dataset=prefix,
cnn_index=n)
else:
# Read/localize the data
test_dataset = return_dataset(model_options.mode,
caps_dir,
test_df,
model_options.preprocessing,
train_transformations=None,
all_transformations=all_transforms,
params=model_options,
labels=labels)
# Load the data
test_loader = DataLoader(test_dataset,
batch_size=model_options.batch_size,
shuffle=False,
num_workers=model_options.nproc,
pin_memory=True)
# Load model from path
model = create_model(model_options, test_dataset.size)
best_model, best_epoch = load_model(model,
join(model_path, selection),
gpu,
filename='model_best.pth.tar')
# Run the model on the data
predictions_df, metrics = test(best_model,
test_loader,
gpu,
criterion,
mode=model_options.mode,
use_labels=labels)
if labels:
logger.info(
"%s level %s balanced accuracy is %f for model selected on %s"
% (model_options.mode, prefix, metrics["balanced_accuracy"],
selection))
mode_level_to_tsvs(output_dir,
predictions_df,
metrics,
fold,
selection,
model_options.mode,
dataset=prefix)
def inference_from_model_generic(caps_dir,
tsv_path,
model_path,
model_options,
num_cnn=None,
selection="best_balanced_accuracy"):
'''
Inference using an image/subject CNN model
'''
from os.path import join
gpu = not model_options.use_cpu
# Recreate the model with the network described in the json file
# Initialize the model
if model_options.model == 'UNet3D':
print('********** init UNet3D model for test! **********')
model = create_model(model_options.model,
gpu=model_options.gpu,
dropout=model_options.dropout,
device_index=model_options.device,
in_channels=model_options.in_channels,
out_channels=model_options.out_channels,
f_maps=model_options.f_maps,
layer_order=model_options.layer_order,
num_groups=model_options.num_groups,
num_levels=model_options.num_levels)
elif model_options.model == 'ResidualUNet3D':
print('********** init ResidualUNet3D model for test! **********')
model = create_model(model_options.model,
gpu=model_options.gpu,
dropout=model_options.dropout,
device_index=model_options.device,
in_channels=model_options.in_channels,
out_channels=model_options.out_channels,
f_maps=model_options.f_maps,
layer_order=model_options.layer_order,
num_groups=model_options.num_groups,
num_levels=model_options.num_levels)
elif model_options.model == 'UNet3D_add_more_fc':
print('********** init UNet3D_add_more_fc model for test! **********')
model = create_model(model_options.model,
gpu=model_options.gpu,
dropout=model_options.dropout,
device_index=model_options.device,
in_channels=model_options.in_channels,
out_channels=model_options.out_channels,
f_maps=model_options.f_maps,
layer_order=model_options.layer_order,
num_groups=model_options.num_groups,
num_levels=model_options.num_levels)
elif model_options.model == 'ResidualUNet3D_add_more_fc':
print(
'********** init ResidualUNet3D_add_more_fc model for test! **********'
)
model = create_model(model_options.model,
gpu=model_options.gpu,
dropout=model_options.dropout,
device_index=model_options.device,
in_channels=model_options.in_channels,
out_channels=model_options.out_channels,
f_maps=model_options.f_maps,
layer_order=model_options.layer_order,
num_groups=model_options.num_groups,
num_levels=model_options.num_levels)
elif model_options.model == 'VoxCNN':
print('********** init VoxCNN model for test! **********')
model = create_model(model_options.model,
gpu=model_options.gpu,
device_index=model_options.device)
elif model_options.model == 'ConvNet3D':
print('********** init ConvNet3D model for test! **********')
model = create_model(model_options.model,
gpu=model_options.gpu,
device_index=model_options.device)
elif 'gcn' in model_options.model:
print('********** init {}-{} model for test! **********'.format(
model_options.model, model_options.gnn_type))
model = create_model(
model_options.model,
gpu=model_options.gpu,
device_index=model_options.device,
gnn_type=model_options.gnn_type,
gnn_dropout=model_options.gnn_dropout,
gnn_dropout_adj=model_options.gnn_dropout_adj,
gnn_non_linear=model_options.gnn_non_linear,
gnn_undirected=model_options.gnn_undirected,
gnn_self_loop=model_options.gnn_self_loop,
gnn_threshold=model_options.gnn_threshold,
)
elif model_options.model == 'ROI_GCN':
print('********** init ROI_GCN model for test! **********')
model = create_model(
model_options.model,
gpu=model_options.gpu,
device_index=model_options.device,
gnn_type=model_options.gnn_type,
gnn_dropout=model_options.gnn_dropout,
gnn_dropout_adj=model_options.gnn_dropout_adj,
gnn_non_linear=model_options.gnn_non_linear,
gnn_undirected=model_options.gnn_undirected,
gnn_self_loop=model_options.gnn_self_loop,
gnn_threshold=model_options.gnn_threshold,
nodel_vetor_layer=model_options.nodel_vetor_layer,
classify_layer=model_options.classify_layer,
num_node_features=model_options.num_node_features,
num_class=model_options.num_class,
roi_size=model_options.roi_size,
num_nodes=model_options.num_nodes,
gnn_pooling_layers=model_options.gnn_pooling_layers,
global_sort_pool_k=model_options.global_sort_pool_k,
layers=model_options.layers,
shortcut_type=model_options.shortcut_type,
use_nl=model_options.use_nl,
dropout=model_options.dropout,
device=model_options.device)
elif model_options.model == 'SwinTransformer3d':
print('********** init SwinTransformer3d model for test! **********')
model = create_model(
model_options.model,
gpu=model_options.gpu,
dropout=model_options.dropout,
device_index=model_options.device,
sw_patch_size=model_options.sw_patch_size,
window_size=model_options.window_size,
mlp_ratio=model_options.mlp_ratio,
drop_rate=model_options.drop_rate,
attn_drop_rate=model_options.attn_drop_rate,
drop_path_rate=model_options.drop_path_rate,
qk_scale=model_options.qk_scale,
embed_dim=model_options.embed_dim,
depths=model_options.depths,
num_heads=model_options.num_heads,
qkv_bias=model_options.qkv_bias,
ape=model_options.ape,
patch_norm=model_options.patch_norm,
)
else:
model = create_model(
model_options.model,
gpu=model_options.gpu,
dropout=model_options.dropout,
device_index=model_options.device,
)
transformations = get_transforms(model_options.mode,
model_options.minmaxnormalization)
# Define loss and optimizer
criterion = nn.CrossEntropyLoss()
if model_options.mode_task == 'multicnn':
predictions_df = pd.DataFrame()
metrics_df = pd.DataFrame()
for n in range(num_cnn):
dataset = return_dataset(model_options.mode,
caps_dir,
tsv_path,
model_options.preprocessing,
transformations,
model_options,
cnn_index=n)
print('test data size:{}'.format(len(dataset)))
test_loader = DataLoader(dataset,
batch_size=model_options.batch_size,
shuffle=False,
num_workers=model_options.nproc,
pin_memory=True,
drop_last=model_options.drop_last)
# load the best trained model during the training
model, best_epoch = load_model(model,
join(model_path, 'cnn-%i' % n,
selection),
gpu,
filename='model_best.pth.tar',
device_index=model_options.device)
cnn_df, cnn_metrics = test(model,
test_loader,
gpu,
criterion,
mode=model_options.mode,
device_index=model_options.device)
predictions_df = pd.concat([predictions_df, cnn_df])
metrics_df = pd.concat(
[metrics_df, pd.DataFrame(cnn_metrics, index=[0])])
predictions_df.reset_index(drop=True, inplace=True)
metrics_df.reset_index(drop=True, inplace=True)
else:
# Load model from path
best_model, best_epoch = load_model(model,
join(model_path, selection),
gpu,
filename='model_best.pth.tar',
device_index=model_options.device)
# Read/localize the data
data_to_test = return_dataset(model_options.mode, caps_dir, tsv_path,
model_options.preprocessing,
transformations, model_options)
print('test data size:{}'.format(len(data_to_test)))
# Load the data
test_loader = DataLoader(data_to_test,
batch_size=model_options.batch_size,
shuffle=False,
num_workers=model_options.nproc,
pin_memory=True,
drop_last=model_options.drop_last)
# Run the model on the data
predictions_df, metrics = test(best_model,
test_loader,
gpu,
criterion,
mode=model_options.mode,
device_index=model_options.device,
model_options=model_options)
metrics_df = pd.DataFrame(metrics, index=[0])
return predictions_df, metrics_df