def evaluate():
net.eval()
test_loss = 0
targets, outputs = [], []
with torch.no_grad():
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
test_loss += batch_loss
test_loss /= (batch_id + 1)
test_acc = compute_accuracy(torch.cat(targets), torch.cat(outputs))
targets_temp = torch.cat(targets).cpu().numpy()
outputs_temp = np.argmax(torch.cat(outputs).cpu().numpy(), axis=1)
log_string('Glomerulus Level Classification Confusion Matrix and Accuracy: ')
log_string(str(confusion_matrix(targets_temp, outputs_temp)))
# display
log_string("validation_loss: {0:.4f}, validation_accuracy: {1:.02%}"
.format(test_loss, test_acc))
return outputs, targets, test_loss
def evaluate(**kwargs):
best_loss = kwargs['best_loss']
best_acc = kwargs['best_acc']
global_step = kwargs['global_step']
capsule_net.eval()
test_loss = 0
targets, predictions = [], []
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
target = F.one_hot(target, options.num_classes)
y_pred, x_reconst, v_length = capsule_net(data)
loss = capsule_loss(data, target, v_length, x_reconst)
targets += [target]
predictions += [y_pred]
test_loss += loss
test_loss /= (batch_id + 1)
test_acc = compute_accuracy(torch.cat(targets), torch.cat(predictions))
# check for improvement
loss_str, acc_str = '', ''
if test_loss <= best_loss:
loss_str, best_loss = '(improved)', test_loss
if test_acc >= best_acc:
acc_str, best_acc = '(improved)', test_acc
# display
log_string(
"validation_loss: {0:.4f} {1}, validation_accuracy: {2:.02%} {3}".
format(test_loss, loss_str, test_acc, acc_str))
# write to TensorBoard
info = {'loss': test_loss, 'accuracy': test_acc}
for tag, value in info.items():
test_logger.scalar_summary(tag, value, global_step)
# save checkpoint model
state_dict = capsule_net.state_dict()
for key in state_dict.keys():
state_dict[key] = state_dict[key].cpu()
save_path = os.path.join(model_dir, '{}.ckpt'.format(global_step))
torch.save(
{
'global_step': global_step,
'loss': test_loss,
'acc': test_acc,
'save_dir': model_dir,
'state_dict': state_dict
}, save_path)
log_string('Model saved at: {}'.format(save_path))
log_string('--' * 30)
return best_loss, best_acc
def evaluate(**kwargs):
best_loss = kwargs['best_loss']
best_acc = kwargs['best_acc']
global_step = kwargs['global_step']
net.eval()
test_loss = 0
targets, outputs = [], []
with torch.no_grad():
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
test_loss += batch_loss
test_loss /= (batch_id + 1)
test_acc = compute_accuracy(torch.cat(targets), torch.cat(outputs))
# check for improvement
loss_str, acc_str = '', ''
if test_loss <= best_loss:
loss_str, best_loss = '(improved)', test_loss
if test_acc >= best_acc:
acc_str, best_acc = '(improved)', test_acc
# display
log_string(
"validation_loss: {0:.4f} {1}, validation_accuracy: {2:.02%}{3}".
format(test_loss, loss_str, test_acc, acc_str))
# write to TensorBoard
info = {'loss': test_loss, 'accuracy': test_acc}
for tag, value in info.items():
test_logger.scalar_summary(tag, value, global_step)
# save checkpoint model
state_dict = net.state_dict()
for key in state_dict.keys():
state_dict[key] = state_dict[key].cpu()
save_path = os.path.join(model_dir, '{}.ckpt'.format(global_step))
torch.save(
{
'global_step': global_step,
'loss': test_loss,
'acc': test_acc,
'save_dir': model_dir,
'state_dict': state_dict
}, save_path)
log_string('Model saved at: {}'.format(save_path))
log_string('--' * 30)
return best_loss, best_acc
def train():
global_step = 0
best_loss = 100
best_acc = 0
for epoch in range(options.epochs):
log_string('**' * 30)
log_string('Training Epoch %03d, Learning Rate %g' %
(epoch + 1, optimizer.param_groups[0]['lr']))
net.train()
train_loss = 0
targets, outputs = [], []
for batch_id, (data, target) in enumerate(train_loader):
global_step += 1
data = data.cuda()
target = target.cuda()
# Forward pass
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
train_loss += batch_loss.item()
# Backward and optimize
optimizer.zero_grad()
batch_loss.backward()
optimizer.step()
if (batch_id + 1) % options.disp_freq == 0:
train_loss /= options.disp_freq
train_acc = compute_accuracy(torch.cat(targets),
torch.cat(outputs))
log_string(
"epoch: {0}, step: {1}, train_loss: {2:.4f} train_accuracy: {3:.02%}"
.format(epoch + 1, batch_id + 1, train_loss, train_acc))
info = {'loss': train_loss, 'accuracy': train_acc}
for tag, value in info.items():
train_logger.scalar_summary(tag, value, global_step)
train_loss = 0
targets, outputs = [], []
if (batch_id + 1) % options.val_freq == 0:
log_string('--' * 30)
log_string('Evaluating at step #{}'.format(global_step))
best_loss, best_acc = evaluate(best_loss=best_loss,
best_acc=best_acc,
global_step=global_step)
net.train()
def train():
global_step = 0
best_loss = 100
best_acc = 0
for epoch in range(options.epochs):
log_string('**' * 30)
log_string('Training Epoch %03d, Learning Rate %g' %
(epoch + 1, optimizer.param_groups[0]['lr']))
capsule_net.train()
train_loss = 0
targets, predictions = [], []
for batch_id, (data, target) in enumerate(train_loader):
data, target = data.cuda(), target.cuda()
global_step += 1
target = F.one_hot(target, options.num_classes)
y_pred, x_reconst, v_length = capsule_net(data, target)
loss = capsule_loss(data, target, v_length, x_reconst)
optimizer.zero_grad()
loss.backward()
optimizer.step()
targets += [target]
predictions += [y_pred]
train_loss += loss.item()
if (batch_id + 1) % options.disp_freq == 0:
train_loss /= options.disp_freq
train_acc = compute_accuracy(torch.cat(targets),
torch.cat(predictions))
log_string(
"epoch: {0}, step: {1}, train_loss: {2:.4f} train_accuracy: {3:.02%}"
.format(epoch + 1, batch_id + 1, train_loss, train_acc))
info = {'loss': train_loss, 'accuracy': train_acc}
for tag, value in info.items():
train_logger.scalar_summary(tag, value, global_step)
train_loss = 0
targets, predictions = [], []
if (batch_id + 1) % options.val_freq == 0:
log_string('--' * 30)
log_string('Evaluating at step #{}'.format(global_step))
best_loss, best_acc = evaluate(best_loss=best_loss,
best_acc=best_acc,
global_step=global_step)
capsule_net.train()
def evaluate():
net.eval()
test_loss = 0
targets, outputs = [], []
with torch.no_grad():
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
test_loss += batch_loss
test_loss /= (batch_id + 1)
test_acc = compute_accuracy(torch.cat(targets), torch.cat(outputs))
# display
log_string(
"validation_loss: {0:.4f}, validation_accuracy: {1:.02%}".format(
test_loss, test_acc))
def evaluate():
capsule_net.eval()
test_loss = 0
targets, predictions = [], []
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
target_ohe = F.one_hot(target, options.num_classes)
y_pred, x_reconst, v_length, c_maps, _ = capsule_net(data, target_ohe)
loss = capsule_loss(data, target_ohe, v_length, x_reconst)
# visualize(data, target, y_pred.argmax(dim=1), c_maps, batch_id)
targets += [target_ohe]
predictions += [y_pred]
test_loss += loss
test_loss /= (len(test_loader) * options.batch_size)
test_acc = compute_accuracy(torch.cat(targets), torch.cat(predictions))
# display
log_string("test_loss: {0:.7f}, test_accuracy: {1:.04%}".format(
test_loss, test_acc))
mean_prob = mc_probs.mean(axis=0)
var_pred_entropy = predictive_entropy(mean_prob)
var_pred_MI = mutual_info(mc_probs)
np.savez_compressed(os.path.join(options.mc_dir, 'output_{}_loos={}_mc={}_GAN={}'.format(DROP,options.loos, options.mc_iter, ST_SET)),
y=iter_targets, data=mc_probs, names=names, subjects=subjects, file_names=file_names, unc=var_pred_entropy)
print('Filename: ' + 'output_{}_loos={}_mciter={}_GAN={}.npz'.format(DROP, options.loos,
options.mc_iter, ST_SET))
acc = 1 - np.count_nonzero(np.not_equal(mean_prob.argmax(axis=1), iter_targets.argmax(axis=1))) / mean_prob.shape[0]
print('accuracy={0:.02%}'.format(acc))
exit()
else:
test_outputs, test_targets, test_loss_ = evaluate()
test_acc = compute_accuracy(torch.cat(test_targets), torch.cat(test_outputs))
targets = torch.cat(test_targets).cpu().numpy()
outputs = np.argmax(torch.cat(test_outputs).cpu().numpy(), axis=1)
# h5f = h5py.File(os.path.join(save_dir, 'prediction_{}.h5'.format(options.loos)), 'w')
# h5f.create_dataset('x', data=x)
# h5f.create_dataset('name', data=names)
# h5f.create_dataset('y', data=targets)
# h5f.create_dataset('y_pred', data=outputs)
# h5f.close()
#################################
# Patient Level classifiers
#################################
# test_pd = pd.DataFrame(list(zip(names, outputs, targets)), columns=['Names', 'Outputs', 'Targets'])
def train():
global_step = 0
best_loss = 100
best_acc = 0
best_auc = 0
for epoch in range(options.epochs):
log_string('**' * 30)
log_string('Training Epoch %03d, Learning Rate %g' %
(epoch + 1, optimizer.param_groups[0]['lr']))
capsule_net.train()
total_loss_ = np.zeros(4)
targets, preds_coarse, preds_fine, preds_drop, preds_dist = [], [], [], [], []
# increments the margin for spread loss
if options.loss_type == 'spread' and (
epoch + 1) % options.n_eps_for_m == 0 and epoch != 0:
capsule_loss.margin += options.m_delta
capsule_loss.margin = min(capsule_loss.margin, options.m_max)
log_string(' *------- Margin increased to {0:.1f}'.format(
capsule_loss.margin))
for batch_id, (data, target, _) in enumerate(train_loader):
global_step += 1
data, target = data.cuda(), target.cuda()
p_coarse, activation_map = capsule_net(data, target)
p_coarse_length = get_vector_length(p_coarse)
loss = capsule_loss(p_coarse_length, target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
targets += [target]
preds_coarse += [p_coarse_length]
total_loss_[0] += loss.item()
##################################
# PEEKABOO Training
##################################
##################################
# Patch Cropping
##################################
with torch.no_grad():
crop_mask = F.upsample_bilinear(
activation_map,
size=(data.size(2), data.size(3))) > options.theta_c
crop_images = []
for batch_index in range(crop_mask.size(0)):
nonzero_indices = torch.nonzero(crop_mask[batch_index, 0,
...])
height_min = nonzero_indices[:, 0].min()
height_max = nonzero_indices[:, 0].max()
width_min = nonzero_indices[:, 1].min()
width_max = nonzero_indices[:, 1].max()
crop_images.append(
F.upsample_bilinear(
data[batch_index:batch_index + 1, :,
height_min:height_max, width_min:width_max],
size=options.img_h))
crop_images = torch.cat(crop_images, dim=0).cuda()
p_fine, _ = capsule_net(crop_images, target)
p_fine_length = get_vector_length(p_fine)
loss = capsule_loss(p_fine_length, target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
preds_fine += [p_fine_length.float()]
total_loss_[1] += loss.item()
##################################
# Patch Dropping
##################################
with torch.no_grad():
drop_mask = F.upsample_bilinear(
activation_map,
size=(data.size(2), data.size(3))) <= options.theta_d
drop_images = data * drop_mask.float()
# drop images forward
p_drop, _ = capsule_net(drop_images.cuda(), target)
p_drop_length = get_vector_length(p_drop)
loss = capsule_loss(p_drop_length, target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
preds_drop += [p_drop_length.float()]
total_loss_[2] += loss.item()
##################################
# Distillation
##################################
p_dist = (p_coarse + p_fine) / 2
p_dist_length = get_vector_length(p_dist)
loss = capsule_loss(p_dist_length, target)
preds_dist += [p_dist_length]
total_loss_[3] += loss.item()
##################################
# Displaying and Validation
##################################
if (batch_id + 1) % options.disp_freq == 0:
total_loss_ = total_loss_ / options.disp_freq
train_acc_coarse = compute_accuracy(torch.cat(preds_coarse),
torch.cat(targets))
train_acc_fine = compute_accuracy(torch.cat(preds_fine),
torch.cat(targets))
train_acc_drop = compute_accuracy(torch.cat(preds_drop),
torch.cat(targets))
train_acc_dist = compute_accuracy(torch.cat(preds_dist),
torch.cat(targets))
log_string(
"epoch: {0}, step: {1}, (Coarse-grained): loss: {2:.4f} acc: {3:.02%}, "
"(Fine-grained): loss: {4:.4f} acc: {5:.02%}, "
"(Drop): loss: {6:.4f} acc: {7:.02%}, "
"(Distilled): loss: {8:.4f} acc: {9:.02%}".format(
epoch + 1, batch_id + 1, total_loss_[0],
train_acc_coarse, total_loss_[1], train_acc_fine,
total_loss_[2], train_acc_drop, total_loss_[3],
train_acc_dist))
info = {
'loss/coarse-grained': total_loss_[0],
'loss/fine-grained': total_loss_[1],
'loss/drop': total_loss_[2],
'loss/distilled': total_loss_[3],
'accuracy/coarse-grained': train_acc_coarse,
'accuracy/fine-grained': train_acc_fine,
'accuracy/drop': train_acc_drop,
'accuracy/distilled': train_acc_dist
}
for tag, value in info.items():
train_logger.scalar_summary(tag, value, global_step)
margin_loss_, reg_loss_, total_loss_ = 0, 0, np.zeros(4)
targets, preds_coarse, preds_fine, preds_drop, preds_dist = [], [], [], [], []
if (batch_id + 1) % options.val_freq == 0:
log_string('--' * 30)
log_string('Evaluating at step #{}'.format(global_step))
best_loss, best_acc, best_auc = evaluate(
best_loss=best_loss,
best_acc=best_acc,
best_auc=best_auc,
global_step=global_step)
capsule_net.train()
def train():
global_step = 0
best_loss = 100
best_acc = 0
best_auc = 0
# Default Parameters
beta = 1e-4
for epoch in range(options.epochs):
log_string('**' * 30)
log_string('Training Epoch %03d, Learning Rate %g' %
(epoch + 1, optimizer.param_groups[0]['lr']))
capsule_net.train()
margin_loss_, reg_loss_, total_loss_ = 0, 0, np.zeros(4)
targets, predictions, predictions_crop, predictions_drop, predictions_combined = [], [], [], [], []
# increments the margin for spread loss
if options.loss_type == 'spread' and (
epoch + 1) % options.n_eps_for_m == 0 and epoch != 0:
capsule_loss.margin += options.m_delta
capsule_loss.margin = min(capsule_loss.margin, options.m_max)
log_string(' *------- Margin increased to {0:.1f}'.format(
capsule_loss.margin))
for batch_id, (data, target) in enumerate(train_loader):
data, target = data.cuda(), target.cuda()
global_step += 1
target_ohe = F.one_hot(target, options.num_classes)
att_reg_loss, rec_loss = 0, 0
optimizer.zero_grad()
y_pred, x_reconst, output, attention_map, feats, attention_maps, out_vec_raw = capsule_net(
data, target_ohe)
cls_loss = capsule_loss(output, target)
if options.add_decoder:
rec_loss = reconst_loss(data, x_reconst)
if options.attention_reg:
feature_matrix = feats[np.arange(len(target)), :, target, :]
att_reg_loss = l2_loss(feature_matrix, feature_center[target])
# Update Feature Center
feature_center[target] += beta * (feature_matrix.detach() -
feature_center[target])
total_loss = cls_loss + options.lambda_one * att_reg_loss + options.alpha * rec_loss
total_loss.backward()
optimizer.step()
# Update Feature Center
feature_center[target] += beta * (feature_matrix.detach() -
feature_center[target])
targets += [target_ohe]
predictions += [y_pred]
margin_loss_ += cls_loss.item()
reg_loss_ += att_reg_loss.item()
total_loss_[0] += total_loss.item()
##################################
# Attention Cropping
##################################
empty_map_count = 0
one_nonzero_count = 0
width_count = 0
height_count = 0
with torch.no_grad():
crop_mask = F.upsample_bilinear(
attention_map, size=(data.size(2), data.size(3))) > theta_c
crop_images = []
for batch_index in range(crop_mask.size(0)):
if torch.sum(crop_mask[batch_index]) == 0:
height_min, width_min = 0, 0
height_max, width_max = 200, 200
# print('0, batch: {}, map: {}'.format(batch_index, map_index))
empty_map_count += 1
else:
nonzero_indices = torch.nonzero(crop_mask[batch_index,
0, ...])
if nonzero_indices.size(0) == 1:
height_min, width_min = 0, 0
height_max, width_max = 200, 200
# print('1, batch: {}, map: {}'.format(batch_index, map_index))
one_nonzero_count += 1
else:
height_min = nonzero_indices[:, 0].min()
height_max = nonzero_indices[:, 0].max()
width_min = nonzero_indices[:, 1].min()
width_max = nonzero_indices[:, 1].max()
if width_min == width_max:
if width_min == 0:
width_max += 1
else:
width_min -= 1
width_count += 1
if height_min == height_max:
if height_min == 0:
height_max += 1
else:
height_min -= 1
height_count += 1
crop_images.append(
F.upsample_bilinear(
data[batch_index:batch_index + 1, :,
height_min:height_max, width_min:width_max],
size=options.img_h))
# print('Batch {} : empty map: {}, one nonzero idx: {}, width_issue: {}, height_issue: {}'
# .format(i, empty_map_count, one_nonzero_count, width_count, height_count))
crop_images = torch.cat(crop_images, dim=0).cuda()
# crop images forward
y_pred_crop, _, output_crop, _, _, _, _ = capsule_net(
crop_images, target_ohe)
loss = capsule_loss(output_crop, target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
predictions_crop += [y_pred_crop.float()]
total_loss_[1] += loss.item()
##################################
# Attention Combining
##################################
# final prediction
output_combined = (output + output_crop) / 2
_, y_pred_combined = output_combined.max(dim=1)
y_pred_combined_ohe = F.one_hot(y_pred_combined,
options.num_classes)
loss = capsule_loss(output_combined, target)
predictions_combined += [y_pred_combined_ohe]
total_loss_[3] += loss.item()
##################################
# Attention Dropping
##################################
with torch.no_grad():
drop_mask = F.upsample_bilinear(
attention_map,
size=(data.size(2), data.size(3))) <= theta_d
drop_images = data * drop_mask.float()
# drop images forward
y_pred_drop, _, output_drop, _, _, _, _ = capsule_net(
drop_images.cuda(), target_ohe)
loss = capsule_loss(output_drop, target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
predictions_drop += [y_pred_drop.float()]
total_loss_[2] += loss.item()
if (batch_id + 1) % options.disp_freq == 0:
margin_loss_ /= options.disp_freq
reg_loss_ /= options.disp_freq
total_loss_ = total_loss_ / options.disp_freq
train_acc = compute_accuracy(torch.cat(targets),
torch.cat(predictions))
train_acc_crop = compute_accuracy(torch.cat(predictions_crop),
torch.cat(targets))
train_acc_drop = compute_accuracy(torch.cat(predictions_drop),
torch.cat(targets))
train_acc_comb = compute_accuracy(
torch.cat(predictions_combined), torch.cat(targets))
log_string(
"epoch: {0}, step: {1}, (Raw): loss: {2:.4f} acc: {3:.02%}, "
"(Crop): loss: {4:.4f} acc: {5:.02%}, (Drop): loss: {6:.4f} acc: {7:.02%}, "
"(Comb): loss: {8:.4f} acc: {9:.02%}".format(
epoch + 1, batch_id + 1, total_loss_[0], train_acc,
total_loss_[1], train_acc_crop, total_loss_[2],
train_acc_drop, total_loss_[3], train_acc_comb))
info = {
'loss/raw': total_loss_[0],
'loss/crop': total_loss_[1],
'loss/drop': total_loss_[2],
'loss/combined': total_loss_[3],
'accuracy/raw': train_acc,
'accuracy/crop': train_acc_crop,
'accuracy/drop': train_acc_drop,
'accuracy/comb': train_acc_comb
}
for tag, value in info.items():
train_logger.scalar_summary(tag, value, global_step)
margin_loss_, reg_loss_, total_loss_ = 0, 0, np.zeros(4)
targets, predictions, predictions_crop, predictions_drop, predictions_combined = [], [], [], [], []
if (batch_id + 1) % options.val_freq == 0:
log_string('--' * 30)
log_string('Evaluating at step #{}'.format(global_step))
best_loss, best_acc, best_auc = evaluate(
best_loss=best_loss,
best_acc=best_acc,
best_auc=best_auc,
global_step=global_step)
capsule_net.train()