def val_epoch(model, loader, mel_idx, is_ext=None, n_test=1, get_output=False):
model.eval()
val_loss = []
LOGITS = []
PROBS = []
TARGETS = []
with torch.no_grad():
for (data, target) in tqdm(loader):
if args.use_meta:
data, meta = data
data, meta, target = data.to(device), meta.to(
device), target.to(device)
logits = torch.zeros((data.shape[0], args.out_dim)).to(device)
probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
for I in range(n_test):
l = model(get_trans(data, I), meta)
logits += l
probs += l.softmax(1)
else:
data, target = data.to(device), target.to(device)
logits = torch.zeros((data.shape[0], args.out_dim)).to(device)
probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
for I in range(n_test):
l = model(get_trans(data, I))
logits += l
probs += l.softmax(1)
logits /= n_test
probs /= n_test
LOGITS.append(logits.detach().cpu())
PROBS.append(probs.detach().cpu())
TARGETS.append(target.detach().cpu())
loss = criterion(logits, target)
val_loss.append(loss.detach().cpu().numpy())
val_loss = np.mean(val_loss)
LOGITS = torch.cat(LOGITS).numpy()
PROBS = torch.cat(PROBS).numpy()
TARGETS = torch.cat(TARGETS).numpy()
# if get_output:
# return LOGITS, PROBS
# else:
acc = (PROBS.argmax(1) == TARGETS).mean() * 100.
auc = roc_auc_score((TARGETS == mel_idx).astype(float), PROBS[:, mel_idx])
auc_20 = roc_auc_score((TARGETS[is_ext == 0] == mel_idx).astype(float),
PROBS[is_ext == 0, mel_idx])
#
bal_acc = compute_balanced_accuracy_score(PROBS, TARGETS)
bal_acc_20 = compute_balanced_accuracy_score(PROBS[is_ext == 0],
TARGETS[is_ext == 0])
return LOGITS, PROBS, val_loss, acc, auc, auc_20, bal_acc, bal_acc_20
def main():
df, df_test, meta_features, n_meta_features, mel_idx = get_df(
args.kernel_type, args.out_dim, args.data_dir_2020, args.data_dir_2019,
args.data_dir_2018, args.use_meta)
transforms_train, transforms_val = get_transforms(args.image_size)
if args.DEBUG:
df_test = df_test.sample(args.batch_size * 3)
dataset_test = MelanomaDataset(df_test,
'test',
meta_features,
transform=transforms_val)
test_loader = torch.utils.data.DataLoader(dataset_test,
batch_size=args.batch_size,
num_workers=args.num_workers)
# load model
models = []
for fold in range(1):
if args.eval == 'best':
model_file = os.path.join(
args.model_dir, f'{args.kernel_type}_best_fold{fold}.pth')
elif args.eval == 'best_20':
model_file = os.path.join(
args.model_dir, f'{args.kernel_type}_best_20_fold{fold}.pth')
if args.eval == 'final':
model_file = os.path.join(
args.model_dir, f'{args.kernel_type}_final_fold{fold}.pth')
model = ModelClass(
args.enet_type,
n_meta_features=n_meta_features,
n_meta_dim=[int(nd) for nd in args.n_meta_dim.split(',')],
out_dim=args.out_dim,
pretrained=True,
# meta_model=args.meta_model
)
model = model.to(device)
try: # single GPU model_file
model.load_state_dict(torch.load(model_file), strict=True)
except: # multi GPU model_file
state_dict = torch.load(model_file)
state_dict = {
k[7:] if k.startswith('module.') else k: state_dict[k]
for k in state_dict.keys()
}
model.load_state_dict(state_dict, strict=True)
if len(os.environ['CUDA_VISIBLE_DEVICES']) > 1:
model = torch.nn.DataParallel(model)
model.eval()
models.append(model)
# predict
PROBS = []
with torch.no_grad():
for (data) in tqdm(test_loader):
if args.use_meta:
data, meta = data
data, meta = data.to(device), meta.to(device)
probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
for model in models:
for I in range(args.n_test):
l = model(get_trans(data, I), meta)
probs += l.softmax(1)
else:
data = data.to(device)
probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
for model in models:
for I in range(args.n_test):
l = model(get_trans(data, I))
probs += l.softmax(1)
probs /= args.n_test
probs /= len(models)
PROBS.append(probs.detach().cpu())
PROBS = torch.cat(PROBS).numpy()
# save cvs
df_test['target'] = PROBS[:, mel_idx]
df_test['image_name'] = df_test['image']
df_test[['image_name', 'target'
]].to_csv(os.path.join(args.sub_dir,
f'sub_{args.kernel_type}_{args.eval}.csv'),
index=False)
def predict_image(image_path):
OUTPUTS = []
n_test = 8
transforms_train, transforms_val = get_transforms(config.image_size)
dataset_test = MelanomaDataset(None,
'test',
None,
transform=transforms_val,
image_path=image_path)
test_loader = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
num_workers=0)
for fold in range(5):
model_file = os.path.join(
config.model_dir, f'{config.kernel_type}_best_o_fold{fold}.pth')
ModelClass = Effnet_Melanoma
model = ModelClass(config.enet_type, out_dim=config.out_dim)
model = model.to(config.device)
try: # single GPU model_file
model.load_state_dict(torch.load(model_file,
map_location=config.device),
strict=True)
except: # multi GPU model_file
state_dict = torch.load(model_file, map_location=config.device)
state_dict = {
k[7:] if k.startswith('module.') else k: state_dict[k]
for k in state_dict.keys()
}
model.load_state_dict(state_dict, strict=True)
model.eval()
LOGITS = []
PROBS = []
with torch.no_grad():
for (data) in tqdm(test_loader):
if config.use_meta:
data, meta = data
data, meta = data.to(config.device), meta.to(config.device)
logits = torch.zeros(
(data.shape[0], config.out_dim)).to(config.device)
probs = torch.zeros(
(data.shape[0], config.out_dim)).to(config.device)
for I in range(n_test):
l = model(get_trans(data, I), meta)
logits += l
probs += l.softmax(1)
else:
data = data.to(config.device)
logits = torch.zeros(
(data.shape[0], config.out_dim)).to(config.device)
probs = torch.zeros(
(data.shape[0], config.out_dim)).to(config.device)
for I in range(n_test):
l = model(get_trans(data, I))
logits += l
probs += l.softmax(1)
logits /= n_test
probs /= n_test
LOGITS.append(logits.detach().cpu())
PROBS.append(probs.detach().cpu())
LOGITS = torch.cat(LOGITS).numpy()
PROBS = torch.cat(PROBS).numpy()
OUTPUTS.append(PROBS[:, config.mel_idx])
#If you are predicting on your own moles, your don't need to rank the probability
pred = np.zeros(OUTPUTS[0].shape[0])
for probs in OUTPUTS:
pred += probs
pred /= len(OUTPUTS)
return round(pred[0], 8)
def main():
df, df_test, meta_features, n_meta_features, mel_idx = get_df(
args.kernel_type, args.out_dim, args.data_dir, args.data_folder,
args.use_meta)
transforms_train, transforms_val = get_transforms(args.image_size)
if args.DEBUG:
df_test = df_test.sample(args.batch_size * 3)
dataset_test = MelanomaDataset(df_test,
'test',
meta_features,
transform=transforms_val)
test_loader = torch.utils.data.DataLoader(dataset_test,
batch_size=args.batch_size,
num_workers=args.num_workers)
print(f'\nPredicting test set using {args.enet_type} ...')
OUTPUTS = []
for fold in range(5):
if args.eval == 'best':
model_file = os.path.join(
args.model_dir, f'{args.kernel_type}_best_fold{fold}.pth')
elif args.eval == 'best_20':
model_file = os.path.join(
args.model_dir, f'{args.kernel_type}_best_20_fold{fold}.pth')
if args.eval == 'final':
model_file = os.path.join(
args.model_dir, f'{args.kernel_type}_final_fold{fold}.pth')
model = ModelClass(
args.enet_type,
n_meta_features=n_meta_features,
n_meta_dim=[int(nd) for nd in args.n_meta_dim.split(',')],
out_dim=args.out_dim)
model = model.to(device)
try: # single GPU model_file
model.load_state_dict(torch.load(model_file), strict=True)
except: # multi GPU model_file
state_dict = torch.load(model_file)
state_dict = {
k[7:] if k.startswith('module.') else k: state_dict[k]
for k in state_dict.keys()
}
model.load_state_dict(state_dict, strict=True)
if len(os.environ['CUDA_VISIBLE_DEVICES']) > 1:
model = torch.nn.DataParallel(model)
model.eval()
PROBS = []
with torch.no_grad():
for (data) in tqdm(test_loader):
if args.use_meta:
data, meta = data
data, meta = data.to(device), meta.to(device)
probs = torch.zeros(
(data.shape[0], args.out_dim)).to(device)
for I in range(args.n_test):
l = model(get_trans(data, I), meta)
probs += l.softmax(1)
else:
data = data.to(device)
probs = torch.zeros(
(data.shape[0], args.out_dim)).to(device)
for I in range(args.n_test):
l = model(get_trans(data, I))
probs += l.softmax(1)
probs /= args.n_test
PROBS.append(probs.detach().cpu())
PROBS = torch.cat(PROBS).numpy()
OUTPUTS.append(PROBS[:, mel_idx])
# Rank per fold (If you are predicting on your own moles, your don't need to rank the probability)
pred = np.zeros(OUTPUTS[0].shape[0])
for probs in OUTPUTS:
pred += pd.Series(probs).rank(pct=True).values
pred /= len(OUTPUTS)
df_test['target'] = pred
df_test[['image_name', 'target'
]].to_csv(os.path.join(args.sub_dir,
f'sub_{args.kernel_type}_{args.eval}.csv'),
index=False)
print('\nSaved submission in -> ./subs')
def main():
df, df_test, meta_features, n_meta_features, mel_idx = get_df(
args.kernel_type,
args.out_dim,
args.data_dir,
args.data_folder,
args.use_meta
)
transforms_train, transforms_val = get_transforms(args.image_size)
if args.DEBUG:
df_test = df_test.sample(args.batch_size * 3)
dataset_test = MelanomaDataset(df_test, 'test', meta_features, transform=transforms_val)
test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, num_workers=args.num_workers)
PROBS = [] ## ! this is defined twice?
for fold in range(5): # ! use model built from each fold
if args.eval == 'best': # ! default
model_file = os.path.join(args.model_dir, f'{args.kernel_type}_best_fold{fold}.pth')
elif args.eval == 'best_20':
model_file = os.path.join(args.model_dir, f'{args.kernel_type}_best_20_fold{fold}.pth')
if args.eval == 'final':
model_file = os.path.join(args.model_dir, f'{args.kernel_type}_final_fold{fold}.pth')
model = ModelClass(
args.enet_type,
n_meta_features=n_meta_features,
n_meta_dim=[int(nd) for nd in args.n_meta_dim.split(',')],
out_dim=args.out_dim
)
model = model.to(device)
try: # single GPU model_file
model.load_state_dict(torch.load(model_file), strict=True)
except: # multi GPU model_file
state_dict = torch.load(model_file)
state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}
model.load_state_dict(state_dict, strict=True)
if len(os.environ['CUDA_VISIBLE_DEVICES']) > 1:
model = torch.nn.DataParallel(model)
model.eval()
PROBS = [] ## ! this is defined twice?
with torch.no_grad():
for (data) in tqdm(test_loader):
if args.use_meta:
data, meta = data
data, meta = data.to(device), meta.to(device)
probs = torch.zeros((data.shape[0], args.out_dim)).to(device) # batch x label
for I in range(args.n_test): # ! fliping images 8 times.
l = model(get_trans(data, I), meta)
probs += l.softmax(1)
else:
data = data.to(device)
probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
for I in range(args.n_test):
l = model(get_trans(data, I))
probs += l.softmax(1)
probs /= args.n_test # ! average over all the flips
PROBS.append(probs.detach().cpu()) ## append prediction for this batch
PROBS = torch.cat(PROBS).numpy() ## put in numpy format, PROBS is total_obs_size x num_labels
df_test['target'] = PROBS[:, mel_idx] # ! takes @mel_idx column
df_test[['image_name', 'target']].to_csv(os.path.join(args.sub_dir, f'sub_{args.kernel_type}_{args.eval}.csv'), index=False)
def main():
df_train, df_test = get_df(args.kernel_type, args.out_dim, args.data_dir)
folds = [int(i) for i in args.val_fold.split(',')]
if args.mode == 'val':
df_test = df_train[df_train['fold'].isin(folds)]
_, transforms_val = get_transforms(args.image_size)
if args.DEBUG:
df_test = df_test.sample(args.batch_size * 3)
dataset_test = RetinalDataset(df_test, 'val', transform=transforms_val)
test_loader = torch.utils.data.DataLoader(dataset_test,
batch_size=args.batch_size,
num_workers=args.num_workers)
# nums = dataset_test.get_num()
# content = f'total num of test:{len(dataset_test)},class nums:{nums}'
# print(content)
# load model
models = []
for fold in range(1):
if args.eval == 'best':
model_file = os.path.join(args.model_dir,
f'{args.kernel_type}_best.pth')
elif args.eval == 'final':
model_file = os.path.join(args.model_dir,
f'{args.kernel_type}_final.pth')
model = ModelClass(args.enet_type,
out_dim=args.out_dim,
pretrained=True,
freeze_cnn=args.freeze_cnn)
model = model.to(device)
try: # single GPU model_file
model.load_state_dict(torch.load(model_file), strict=True)
except: # multi GPU model_file
state_dict = torch.load(model_file)
state_dict = {
k[7:] if k.startswith('module.') else k: state_dict[k]
for k in state_dict.keys()
}
model.load_state_dict(state_dict, strict=True)
# if len(os.environ['CUDA_VISIBLE_DEVICES']) > 1:
# model = torch.nn.DataParallel(model)
model.eval()
models.append(model)
LOGITS = []
with torch.no_grad():
for (data, target) in tqdm(test_loader):
data, target = data.to(device), target.to(device)
logits = torch.zeros((data.shape[0], args.out_dim)).to(device)
for model in models:
for I in range(args.n_test):
l = model(get_trans(data, I))
logits += l
logits /= args.n_test
logits /= len(models)
logits = F.sigmoid(logits)
LOGITS.append(logits.detach().cpu())
LOGITS = torch.cat(LOGITS).numpy()
PREDICT = []
for logit in LOGITS:
predicted_label = []
for i in range(logit.size):
if logit[i].item() > 0.5: # Threshold is 0.5
predicted_label.append(i)
if len(predicted_label) == 0:
predicted_label.append(0)
PREDICT.append(" ".join([str(label) for label in predicted_label]))
df_test['predicted'] = PREDICT
df_test[['filename', 'predicted'
]].to_csv(os.path.join(args.sub_dir,
f'sub_{args.kernel_type}_{args.eval}.csv'),
index=False)
