def evaluate(model, dataset, tokenizer, collator, opt):
sampler = SequentialSampler(dataset)
dataloader = DataLoader(dataset,
sampler=sampler,
batch_size=opt.per_gpu_batch_size,
drop_last=False,
num_workers=10,
collate_fn=collator
)
model.eval()
total = 0
exactmatch = []
model = model.module if hasattr(model, "module") else model
with torch.no_grad():
for i, batch in enumerate(dataloader):
(idx, _, _, context_ids, context_mask) = batch
outputs = model.generate(
input_ids=context_ids.cuda(),
attention_mask=context_mask.cuda(),
max_length=50
)
for k, o in enumerate(outputs):
ans = tokenizer.decode(o, skip_special_tokens=True)
gold = dataset.get_example(idx[k])['answers']
score = src.evaluation.ems(ans, gold)
total += 1
exactmatch.append(score)
exactmatch, total = src.util.weighted_average(np.mean(exactmatch), total, opt)
return exactmatch
def evaluate(model, dataset, dataloader, tokenizer, opt):
loss, curr_loss = 0.0, 0.0
model.eval()
if hasattr(model, "module"):
model = model.module
if opt.write_crossattention_scores:
model.overwrite_forward_crossattention()
model.reset_score_storage()
total = 0
exactmatch = []
if opt.write_results:
write_path = Path(opt.checkpoint_dir) / opt.name / 'test_results'
fw = open(write_path / '%d.txt' % opt.global_rank, 'a')
with torch.no_grad():
for i, batch in enumerate(dataloader):
(idx, _, _, context_ids, context_mask) = batch
if opt.write_crossattention_scores:
model.reset_score_storage()
outputs = model.generate(
input_ids=context_ids.cuda(),
attention_mask=context_mask.cuda(),
max_length=50,
)
if opt.write_crossattention_scores:
crossattention_scores = model.get_crossattention_scores(
context_mask.cuda())
for k, o in enumerate(outputs):
ans = tokenizer.decode(o, skip_special_tokens=True)
example = dataset.get_example(idx[k])
question = example['question']
gold = example['answers']
exid = example['id']
score = src.evaluation.ems(ans, gold)
exactmatch.append(score)
if opt.write_results:
fw.write(str(exid) + "\t" + ans + '\n')
if opt.write_crossattention_scores:
ctxs = example['ctxs']
for j in range(context_ids.size(1)):
ctxs[j]['score'] = crossattention_scores[k, j].item()
total += 1
if (i + 1) % opt.eval_print_freq == 0:
log = f'Process rank:{opt.global_rank}, {i+1} / {len(dataloader)}'
log += f' -- average = {np.mean(exactmatch):.3f}'
logger.warning(log)
logger.warning(
f'Process rank:{opt.global_rank}, total {total} -- average = {np.mean(exactmatch):.3f}'
)
if opt.is_distributed:
torch.distributed.barrier()
score, total = src.util.weighted_average(np.mean(exactmatch), total, opt)
return score, total
def embed_questions(opt, data, model, tokenizer):
batch_size = opt.per_gpu_batch_size * opt.world_size
dataset = src.data.Dataset(data)
collator = src.data.Collator(opt.question_maxlength, tokenizer)
dataloader = DataLoader(dataset,
batch_size=batch_size,
drop_last=False,
num_workers=10,
collate_fn=collator)
model.eval()
embedding = []
with torch.no_grad():
for k, batch in enumerate(dataloader):
(idx, _, _, question_ids, question_mask) = batch
output = model.embed_text(
text_ids=question_ids.to(opt.device).view(
-1, question_ids.size(-1)),
text_mask=question_mask.to(opt.device).view(
-1, question_ids.size(-1)),
apply_mask=model.apply_question_mask,
extract_cls=model.extract_cls,
)
embedding.append(output)
embedding = torch.cat(embedding, dim=0)
logger.info(f'Questions embeddings shape: {embedding.size()}')
return embedding.cpu().numpy()
def validation(model, criterion, epoch):
# evaluation mode
model.eval()
transform = transforms.Compose([Normalize(), ToTensor()])
data = dataset.Datasets("../dataset/videos/val/", "../dataset/annotations/",transform)
dataloader = DataLoader(data, batch_size=batch_size, shuffle=True, num_workers=4)
dataset_size = dataloader.dataset.len
optimizer = optim.Adam(model.classifier.parameters(), lr=learning_rate)
running_loss = 0.0
i = 0
# iterate over data
for data in dataloader:
# getting the inputs and labels
x1, x2, y = data['previmg'], data['currimg'], data['currbb']
# wrapping them in Variable
if use_gpu:
x1, x2, y = Variable(x1.cuda()), Variable(x2.cuda()), Variable(y.cuda(), requires_grad=False)
else:
x1, x2, y = Variable(x1), Variable(x2), Variable(y, requires_grad=False)
# zero the parameter gradients
optimizer.zero_grad()
# forward
output = model(x1, x2)
loss = criterion(output, y)
# backward + optimize
loss.backward()
optimizer.step()
running_loss += loss.data.item()
print('Validation epoch : %d, step : %d, loss : %f' % (epoch , i, loss.data.item()))
i = i + 1
val_loss = running_loss/dataset_size
return val_loss
def main(opt):
src.util.init_logger(is_main=True)
tokenizer = transformers.BertTokenizerFast.from_pretrained('bert-base-uncased')
data = src.data.load_data(opt.data)
model_class = src.model.Retriever
model = model_class.from_pretrained(opt.model_path)
model.cuda()
model.eval()
if not opt.no_fp16:
model = model.half()
# index = src.index.Indexer(model.config.indexing_dimension, opt.n_subquantizers, opt.n_bits)
# # index all passages
# input_paths = glob.glob(args.passages_embeddings)
# input_paths = sorted(input_paths)
# embeddings_dir = Path(input_paths[0]).parent
# index_path = embeddings_dir / 'index.faiss'
# if args.save_or_load_index and index_path.exists():
# index.deserialize_from(embeddings_dir)
# else:
# logger.info(f'Indexing passages from files {input_paths}')
# start_time_indexing = time.time()
# index_encoded_data(index, input_paths, opt.indexing_batch_size)
# logger.info(f'Indexing time: {time.time()-start_time_indexing:.1f} s.')
# if args.save_or_load_index:
# index.serialize(embeddings_dir)
questions_embedding, question_ids = embed_questions(opt, data, model, tokenizer)
lout = []
for ct, (qe, qid) in enumerate(zip(questions_embedding, question_ids)):
lout.append((str(ct), (qid, qe)))
pickle.dump(lout, open("fid.pkl", 'wb'))
exit()
# get top k results
start_time_retrieval = time.time()
top_ids_and_scores = index.search_knn(questions_embedding, args.n_docs)
logger.info(f'Search time: {time.time()-start_time_retrieval:.1f} s.')
passages = src.util.load_passages(args.passages)
passages = {x[0]:(x[1], x[2]) for x in passages}
add_passages(data, passages, top_ids_and_scores)
hasanswer = validate(data, args.validation_workers)
add_hasanswer(data, hasanswer)
output_path = Path(args.output_path)
output_path.parent.mkdir(parents=True, exist_ok=True)
with open(args.output_path, 'w') as fout:
json.dump(data, fout, indent=4)
logger.info(f'Saved results to {args.output_path}')
def evaluate(model, dataset, collator, opt):
sampler = SequentialSampler(dataset)
dataloader = DataLoader(dataset,
sampler=sampler,
batch_size=opt.per_gpu_batch_size,
drop_last=False,
num_workers=10,
collate_fn=collator)
model.eval()
if hasattr(model, "module"):
model = model.module
total = 0
eval_loss = []
avg_topk = {k: [] for k in [1, 2, 5] if k <= opt.n_context}
idx_topk = {k: [] for k in [1, 2, 5] if k <= opt.n_context}
inversions = []
with torch.no_grad():
for i, batch in enumerate(dataloader):
(idx, question_ids, question_mask, context_ids, context_mask,
gold_score) = batch
_, _, scores, loss = model(
question_ids=question_ids.cuda(),
question_mask=question_mask.cuda(),
passage_ids=context_ids.cuda(),
passage_mask=context_mask.cuda(),
gold_score=gold_score.cuda(),
)
src.evaluation.eval_batch(scores, inversions, avg_topk, idx_topk)
total += question_ids.size(0)
inversions = src.util.weighted_average(np.mean(inversions), total, opt)[0]
for k in avg_topk:
avg_topk[k] = src.util.weighted_average(np.mean(avg_topk[k]), total,
opt)[0]
idx_topk[k] = src.util.weighted_average(np.mean(idx_topk[k]), total,
opt)[0]
return loss, inversions, avg_topk, idx_topk
def main(opt):
logger = src.util.init_logger(is_main=True)
tokenizer = transformers.BertTokenizerFast.from_pretrained(
'bert-base-uncased')
model_class = src.model.Retriever
#model, _, _, _, _, _ = src.util.load(model_class, opt.model_path, opt)
model = model_class.from_pretrained(opt.model_path)
model.eval()
model = model.to(opt.device)
if not opt.no_fp16:
model = model.half()
passages = src.util.load_passages(args.passages)
shard_size = len(passages) // args.num_shards
start_idx = args.shard_id * shard_size
end_idx = start_idx + shard_size
if args.shard_id == args.num_shards - 1:
end_idx = len(passages)
passages = passages[start_idx:end_idx]
logger.info(
f'Embedding generation for {len(passages)} passages from idx {start_idx} to {end_idx}'
)
allids, allembeddings = embed_passages(opt, passages, model, tokenizer)
output_path = Path(args.output_path)
save_file = output_path.parent / (output_path.name +
f'_{args.shard_id:02d}')
output_path.parent.mkdir(parents=True, exist_ok=True)
logger.info(f'Saving {len(allids)} passage embeddings to {save_file}')
with open(save_file, mode='wb') as f:
pickle.dump((allids, allembeddings), f)
logger.info(
f'Total passages processed {len(allids)}. Written to {save_file}.')
def train():
tot_loss = 0.0
tot_correct = 0
tot_lsl = 0.0
tot_lss_1 = 0.0
tot_lss_2 = 0.0
tot_lsd = 0.0
for inputs, labels in train_loader:
inputs = inputs.to(device)
labels = labels.to(device)
# CutMix regularizer
label_original = F.one_hot(labels, 10)
lam = np.random.beta(cutmix_beta, cutmix_beta)
rand_index = torch.randperm(inputs.size()[0])
x_cutmix = inputs.clone().detach()
x_a = inputs[rand_index, :, :, :]
labels_a = labels[rand_index]
bbx1, bby1, bbx2, bby2 = rand_bbox(inputs.size(), lam)
M = torch.zeros((inputs.size()[-2], inputs.size()[-1]))
M = M.to(device)
M[bbx1:bbx2, bby1:bby2] = 1
x_cutmix[:, :, bbx1:bbx2, bby1:bby2] = x_a[:, :, bbx1:bbx2, bby1:bby2]
lam = ((bbx2 - bbx1) * (bby2 - bby1) /
(inputs.size()[-1] * inputs.size()[-2]))
label_cutmix = lam * label_original[rand_index, :] + (
1 - lam) * label_original
# x_a
model.eval()
with torch.no_grad():
_dummy1, _dummy2, _dummpy3, Y_a = model(x_a)
# CutMix
model.train()
optimizer.zero_grad()
outputs, pool_outputs, M_hat, Y_cutmix = model(x_cutmix)
# Resize M to H0 * W0
M = M.unsqueeze(dim=0).unsqueeze(dim=1)
M = M.repeat(inputs.size()[0], 1, 1, 1)
M_resizer = torch.nn.MaxPool2d(int(M.size()[-1] / M_hat.size()[-1]))
M = M_resizer(M)
lsl = lam * criterion_ce(outputs, labels_a) + (1 - lam) * criterion_ce(
outputs, labels)
lss_1 = criterion_lss1(M_hat, M)
lss_2 = criterion_lss2(M[0, 0, :, :] * Y_cutmix, M[0, 0, :, :] * Y_a)
lsd = criterion_lss2(outputs, pool_outputs.detach()) + 0.5 * (
lam * criterion_ce(pool_outputs, labels_a) +
(1 - lam) * criterion_ce(pool_outputs, labels))
# loss = lsl + lss_1 + lss_2 + lsd
loss = lsl
loss.backward()
optimizer.step()
_, preds = torch.max(outputs, 1)
_, labels = torch.max(label_cutmix, 1)
tot_loss += loss.item() * inputs.size(0)
tot_correct += torch.sum(preds == labels.data).item()
tot_lsl += lsl.item() * inputs.size(0)
tot_lss_1 += lss_1.item() * inputs.size(0)
tot_lss_2 += lss_2.item() * inputs.size(0)
tot_lsd += lsd.item() * inputs.size(0)
len_ = len(train_loader.dataset)
epoch_loss = tot_loss / len_
epoch_acc = tot_correct / len_
epoch_lsl = tot_lsl / len_
epoch_lss_1 = tot_lss_1 / len_
epoch_lss_2 = tot_lss_2 / len_
epoch_lsd = tot_lsd / len_
return epoch_loss, epoch_acc, epoch_lsl, epoch_lss_1, epoch_lss_2, epoch_lsd
model.zero_grad()
# get the output from the model
output, h = model(inputs, h)
# calculate the loss and perform backprop
loss = criterion(output.squeeze(), labels.float())
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
nn.utils.clip_grad_norm_(model.parameters(), clip)
optimizer.step()
if counter % print_every == 0:
# Get validation loss
val_losses = []
model.eval()
for inputs, labels in Valid_loader:
# Creating new variables for the hidden state, otherwise
# we'd backprop through the entire training history
batch_siz = inputs.shape[0]
val_h = model.init_hidden(batch_siz)
val_h = tuple([each.data for each in val_h])
if (train_on_gpu):
inputs, labels = inputs.cuda(), labels.cuda()
output, val_h = model(inputs, val_h)
val_loss = criterion(output.squeeze(), labels.float())
sess_file = "model/RNN_0.01lr_48inputs_50neurons_10000iter"
rnn = RNN(load_model=sess_file + ".param")
mse_vs_iter = pd.DataFrame(rnn.mse)
mse_vs_iter = mse_vs_iter.set_index("iteration")
print(mse_vs_iter)
mse_vs_iter.plot()
plt.show()
"""
#PREDICTION
#sess_file = "model/RNN_0.03lr_122inputs_100neurons_actFct-tanh"
#rnn = RNN(load_model=sess_file + ".param")
y_pred, test_set, mse, input_data = model.prediction(sess_file, rnn, data_handle)
model.plot_prediction(y_pred, test_set, input_data)
plt.show()
"""
"""
sess_file = "model/RNN_0.03lr_122inputs_100neurons_actFct-tanh"
model.eval(sess_file, data_handle, 42, 10)
plt.show()
"""
"""
#PREDICTION BLIND
nb_pred = 16
test_date = datetime.datetime.strptime("2018-10-19_06", "%Y-%m-%d_%H")
input_start_date = test_date - datetime.timedelta(hours=hp.nb_time_step)
input_data, input_scaled = data_handle.get_sample(input_start_date, hp.nb_time_step)
test_set, test_scaled = data_handle.get_sample(test_date, nb_pred)
y_pred = rnn.run(sess_name=sess_name, input_set=input_scaled, nb_pred=nb_pred)
y_pred = data_handle.inverse_transform(y_pred)
"""
def run(args):
print('Task 1: clear cell grade prediction')
path = '/data/larson2/RCC_dl/new/clear_cell/'
transform = {
'train':
transforms.Compose([
transforms.Lambda(lambda x: torch.Tensor(x)),
src.dataloader.Rescale(-160, 240,
zero_center=True), # rset dynamic range
transforms.Lambda(
lambda x: x.repeat(3, 1, 1, 1).permute(3, 0, 1, 2)),
# src.dataloader.Normalize(),
# src.dataloader.Crop(110),
# src.dataloader.RandomCenterCrop(90),
src.dataloader.RandomHorizontalFlip(),
# src.dataloader.RandomRotate(25),
src.dataloader.Resize(256)
]),
'val':
transforms.Compose([
transforms.Lambda(lambda x: torch.Tensor(x)),
src.dataloader.Rescale(-160, 240,
zero_center=True), # rset dynamic range
transforms.Lambda(
lambda x: x.repeat(3, 1, 1, 1).permute(3, 0, 1, 2)),
# src.dataloader.Normalize(),
# src.dataloader.Crop(90),
src.dataloader.Resize(256)
])
}
my_dataset = {
'train':
src.dataloader.RCCDataset_h5(path,
mode='train',
transform=transform['train']),
'val':
src.dataloader.RCCDataset_h5(path,
mode='val',
transform=transform['train'])
}
my_loader = {
x: DataLoader(my_dataset[x], batch_size=1, shuffle=True, num_workers=4)
for x in ['train', 'val']
}
print('train size: ', len(my_loader['train']))
print('train size: ', len(my_loader['val']))
### Some Checkers
print('Summary: ')
print('\ttrain size: ', len(my_loader['train']))
print('\ttrain size: ', len(my_loader['val']))
print('\tDatatype = ', next(iter(my_loader['train']))[0].dtype)
print('\tMin = ', next(iter(my_loader['train']))[0].min())
print('\tMax = ', next(iter(my_loader['train']))[0].max())
print('\tInput size', next(iter(my_loader['train']))[0].shape)
# print('\tweight = ', args.weight)
### Tensorboard Log Setup
log_root_folder = "/data/larson2/RCC_dl/logs/"
now = datetime.now()
now = now.strftime("%Y%m%d-%H%M%S")
logdir = os.path.join(
log_root_folder,
f"{now}_model_{args.model}_{args.prefix_name}_epoch_{args.epochs}_weight_{args.weight}_lr_{args.lr}_gamma_{args.gamma}_lrsche_{args.lr_scheduler}_{now}"
)
# os.makedirs(logdir)
print(f'\tlogdir = {logdir}')
writer = SummaryWriter(logdir)
### Model Selection
device = torch.device(
"cuda:{}".format(args.gpu) if torch.cuda.is_available() else "cpu")
model = src.model.TDNet()
model = model.to(device)
writer.add_graph(model, my_dataset['train'][0][0].to(device))
print('\tCuda:', torch.cuda.is_available(), f'\n\tdevice = {device}')
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=0.1)
if args.lr_scheduler == "plateau":
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
patience=3,
factor=.3,
threshold=1e-4,
verbose=True)
elif args.lr_scheduler == "step":
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=args.gamma)
pos_weight = torch.FloatTensor([args.weight]).to(device)
criterion = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight)
### Ready?
best_val_loss = float('inf')
best_val_auc = float(0)
best_model_wts = copy.deepcopy(model.state_dict())
iteration_change_loss = 0
t_start_training = time.time()
### Here we go
for epoch in range(args.epochs):
current_lr = get_lr(optimizer)
t_start = time.time()
epoch_loss = {'train': 0., 'val': 0.}
epoch_corrects = {'train': 0., 'val': 0.}
epoch_acc = 0.0
epoch_AUC = 0.0
for phase in ['train', 'val']:
if phase == 'train':
if args.lr_scheduler == "step":
scheduler.step()
model.train()
else:
model.eval()
running_losses = []
running_corrects = 0.
y_trues = []
y_probs = []
y_preds = []
print('lr: ', current_lr)
for i, (inputs, labels, header) in enumerate(my_loader[phase]):
optimizer.zero_grad()
inputs = inputs.to(device)
labels = labels.to(device)
# forward
# track history only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs.float()) # raw logits
probs = torch.sigmoid(
outputs) # [0, 1] probability, shape = s * 1
preds = torch.round(
probs
) # 0 or 1, shape = s * 1, prediction for each slice
pt_pred, _ = torch.mode(
preds, 0
) # take majority vote, shape = 1, prediction for each patient
count0 = (preds == 0).sum().float()
count1 = (preds == 1).sum().float()
pt_prob = count1 / (preds.shape[0])
# convert label to slice level
loss = criterion(outputs, labels.repeat(
inputs.shape[1], 1)) # inputs shape = 1*s*3*256*256
# backward + optimize only if in training phases
if phase == 'train':
loss.backward()
optimizer.step()
# multiple loss by slice num per batch?
running_losses.append(loss.item()) # * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
y_trues.append(int(labels.item()))
y_probs.append(pt_prob.item()) # use ratio to get probability
y_preds.append(pt_pred.item())
writer.add_scalar(f'{phase}/Loss', loss.item(),
epoch * len(my_loader[phase]) + i)
writer.add_pr_curve('{phase}pr_curve', y_trues, y_probs, 0)
if (i % args.log_every == 0) & (i > 0):
print(
'Epoch: {0}/{1} | Single batch number : {2}/{3} | avg loss:{4} | Acc: {5:.4f} | lr: {6}'
.format(epoch + 1, args.epochs, i,
len(my_loader[phase]),
np.round(np.mean(running_losses), 4),
(running_corrects / len(my_loader[phase])),
current_lr))
# epoch statistics
epoch_loss[phase] = np.round(np.mean(running_losses), 4)
epoch_corrects[phase] = (running_corrects / len(my_loader[phase]))
cm = confusion_matrix(y_trues, y_preds, labels=[0, 1])
src.helper.print_cm(cm, ['0', '1'])
sens, spec, acc = src.helper.compute_stats(y_trues, y_preds)
print('sens: {:.4f}'.format(sens))
print('spec: {:.4f}'.format(spec))
print('acc: {:.4f}'.format(acc))
print()
print(
'\ Summary train loss: {0} | val loss: {1} | train acc: {2:.4f} | val acc: {3:.4f}'
.format(epoch_loss['train'], epoch_loss['val'],
epoch_corrects['train'], epoch_corrects['val']))
print('-' * 30)
def train_keypoint_rcnn(data=None,
epochs: int = None,
lr: float = 1e-5,
pretrained: str = None):
model = src.model.keypoint_rcnn
if not isinstance(pretrained, str) and pretrained is not None:
raise ValueError(
f'Argument "pretrained" must be a path to a valid mask file, '
f'not {pretrained} with type {type(pretrained)}')
if epochs is None:
epochs = 500
if pretrained is not None:
print('Loading...')
model.load_state_dict(torch.load(pretrained))
if torch.cuda.is_available(): device = 'cuda:0'
else: device = 'cpu'
# tests = KeypointData('/media/DataStorage/Dropbox (Partners HealthCare)/DetectStereocillia/data/keypoint_train_data')
model = model.train().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)
for e in range(epochs):
epoch_loss = []
time_1 = time.clock_gettime_ns(1)
for image, data_dict in data:
for key in data_dict:
data_dict[key] = data_dict[key].to(device)
assert image.shape[1] == 3
optimizer.zero_grad()
loss = model(image.to(device), [data_dict])
losses = 0
for key in loss:
losses += loss[key]
losses.backward()
epoch_loss.append(losses.item())
optimizer.step()
time_2 = time.clock_gettime_ns(1)
delta_time = np.round((np.abs(time_2 - time_1) / 1e9) / 60, decimals=2)
# --------- This is purely to output a nice bar for training --------- #
if e % 5 == 0:
if e > 0:
print('\b \b' * len(out_str), end='')
progress_bar = '[' + '█' * +int(np.round(e / epochs, decimals=1) * 10) + \
' ' * int(
(10 - np.round(e / epochs, decimals=1) * 10)) + f'] {np.round(e / epochs, decimals=3)}%'
out_str = f'epoch: {e} ' + progress_bar + f'| time remaining: {delta_time * (epochs-e)} min | epoch loss: {torch.tensor(epoch_loss).mean().item()}'
print(out_str, end='')
# If its the final epoch print out final string
elif e == epochs - 1:
print('\b \b' * len(out_str), end='')
progress_bar = '[' + '█' * 10 + f'] {1.0}'
out_str = f'epoch: {epochs} ' + progress_bar + f'| time remaining: {0} min | epoch loss: {torch.tensor(epoch_loss).mean().item()}'
print(out_str)
torch.save(model.state_dict(), 'models/keypoint_rcnn.mdl')
model.eval()
out = model(image.unsqueeze(0).cuda())
def evaluate_model(
model,
val_loader,
device,
epoch,
num_epochs,
writer,
current_lr,
log_every=20,
):
_ = model.eval()
model = model.to(device)
y_trues = []
y_logits = []
y_probs = []
y_preds = []
loss_values = []
criterion = torch.nn.BCEWithLogitsLoss()
for i, (image, label, header) in enumerate(val_loader):
image = image.to(device)
label = label.to(device)
outputs = model(image.float())
loss = criterion(outputs, label)
probs = torch.sigmoid(outputs)
preds = torch.round(probs)
loss_values.append(loss.item())
y_trues.append(int(label.item()))
y_logits.append(outputs.item())
y_probs.append(probs.item())
y_preds.append(preds.item())
try:
auc = metrics.roc_auc_score(y_trues, y_probs)
except:
auc = 0.5
writer.add_scalar('Val/Loss', loss.item(), epoch * len(val_loader) + i)
writer.add_scalar('Val/AUC', auc, epoch * len(val_loader) + i)
if (i % log_every == 0) & (i > 0):
print(
'''[Epoch: {0} / {1} |Single batch number : {2} / {3} ] | avg val loss {4} | val auc : {5} | lr : {6}'''
.format(epoch + 1, num_epochs, i, len(val_loader),
np.round(np.mean(loss_values), 4), np.round(auc, 4),
current_lr))
cm = confusion_matrix(y_trues, y_preds, labels=[0, 1])
print_cm(cm, ['0', '1'])
sens, spec, acc = compute_stats(y_trues, y_preds)
print('sens: {:.4f}'.format(sens))
print('spec: {:.4f}'.format(spec))
print('acc: {:.4f}'.format(acc))
print()
writer.add_scalar('Val/AUC_epoch', auc, epoch + i)
val_loss_epoch = np.round(np.mean(loss_values), 4)
val_auc_epoch = np.round(auc, 4)
return val_loss_epoch, val_auc_epoch
