def MC_evaluate(self, dataset='valid', train_step=None):
num_batch = self.num_test_batch if dataset == 'test' else self.num_val_batch
self.sess.run(tf.local_variables_initializer())
y_pred = np.zeros((self.conf.monte_carlo_simulations,
self.conf.val_batch_size, self.conf.num_cls))
mean_pred, var_pred = np.zeros_like(
self.data_reader.y_test), np.zeros_like(self.data_reader.y_test[:,
0])
err = 0.
for step in tqdm(range(num_batch)):
start = self.conf.val_batch_size * step
end = self.conf.val_batch_size * (step + 1)
data_x, data_y = self.data_reader.next_batch(start=start,
end=end,
mode=dataset)
feed_dict = {
self.inputs_pl: data_x,
self.labels_pl: data_y,
self.keep_prob_pl: self.conf.keep_prob,
self.is_train_pl: False
}
for sample_id in range(self.conf.monte_carlo_simulations):
# save predictions from a sample pass
y_pred[sample_id] = self.sess.run(self.y_prob,
feed_dict=feed_dict)
# average and variance over all passes
mean_pred[start:end] = y_pred.mean(axis=0)
var_pred[start:end] = predictive_entropy(mean_pred[start:end])
var_pred[start:end] = mutual_info(y_pred)
# compute batch error
err += np.count_nonzero(
np.not_equal(mean_pred[start:end].argmax(axis=1),
data_y.argmax(axis=1)))
accuracy = 1 - err / self.data_reader.y_test.shape[0]
print('Test accuracy = {0:.02%}'.format(accuracy))
# ST_SET = options.num_gan
# np.savez_compressed(os.path.join(options.mc_dir,
# 'output_{}_loos={}_mciter={}_GAN={}'.format(DROP, options.loos,
# mc_iter+1, ST_SET)), y=iter_targets, data=mc_probs)
# print('Filename: ' + 'output_{}_loos={}_mciter={}_GAN={}.npz'.format(DROP, options.loos,
# mc_iter+1, ST_SET))
# print('accuracy={0:.05%}'.format(acc))
mc_probs = np.concatenate(mc_probs) # [mc_iter, N, C]
if options.GAN == 0:
ST_SET = 0
else:
ST_SET = options.num_gan
mean_prob = mc_probs.mean(axis=0)
var_pred_entropy = predictive_entropy(mean_prob)
var_pred_MI = mutual_info(mc_probs)
np.savez_compressed(os.path.join(options.mc_dir, 'output_{}_loos={}_mc={}_GAN={}'.format(DROP,options.loos, options.mc_iter, ST_SET)),
y=iter_targets, data=mc_probs, names=names, subjects=subjects, file_names=file_names, unc=var_pred_entropy)
print('Filename: ' + 'output_{}_loos={}_mciter={}_GAN={}.npz'.format(DROP, options.loos,
options.mc_iter, ST_SET))
acc = 1 - np.count_nonzero(np.not_equal(mean_prob.argmax(axis=1), iter_targets.argmax(axis=1))) / mean_prob.shape[0]
print('accuracy={0:.02%}'.format(acc))
exit()
else:
test_outputs, test_targets, test_loss_ = evaluate()
test_acc = compute_accuracy(torch.cat(test_targets), torch.cat(test_outputs))
targets = torch.cat(test_targets).cpu().numpy()
#
all_mc_acc = []
for iter_num in range(100):
mean_prob = mc_probs[0:iter_num + 1].mean(axis=0)
all_mc_acc.append(1 - np.count_nonzero(
np.not_equal(mean_prob.argmax(axis=1), y.argmax(axis=1))) /
mean_prob.shape[0])
opt_iter = np.argmax(np.array(all_mc_acc))
# get the values on T
y_probs = mc_probs[:opt_iter] # [opt_iter, N, C]
mean_probs = y_probs.mean(axis=0)
# compute uncertainties
var_pred_entropy = predictive_entropy(mean_probs) # [N, C]
# var_pred_MI = mutual_info(y_probs)
# normalize the uncertainty values
y_var = normalize(var_pred_entropy)
# plot subject level distributions
unq_subs = np.unique(name)
for sub in unq_subs:
if sub not in done_subs:
sub_idx = np.where(name == sub)
sub_x = x[sub_idx]
sub_y = y[sub_idx].argmax(axis=1)
sub_y_pred = mean_probs[sub_idx].argmax(axis=1)
sub_y_prob = np.squeeze(y_probs[:, sub_idx])
sub_y_var = y_var[sub_idx]
def MC_evaluate(self, dataset='valid', train_step=None):
num_batch = self.num_test_batch if dataset == 'test' else self.num_val_batch
hist = np.zeros((self.conf.num_cls, self.conf.num_cls))
self.sess.run(tf.local_variables_initializer())
all_inputs = np.zeros((0, self.conf.height, self.conf.width, self.conf.channel))
all_mask = np.zeros((0, self.conf.height, self.conf.width))
all_pred = np.zeros((0, self.conf.height, self.conf.width))
all_var = np.zeros((0, self.conf.height, self.conf.width))
cls_uncertainty = np.zeros((0, self.conf.height, self.conf.width, self.conf.num_cls))
for step in tqdm(range(num_batch)):
start = self.conf.val_batch_size * step
end = self.conf.val_batch_size * (step + 1)
data_x, data_y = self.data_reader.next_batch(start=start, end=end, mode=dataset)
mask_pred_mc = [np.zeros((self.conf.val_batch_size, self.conf.height, self.conf.width))
for _ in range(self.conf.monte_carlo_simulations)]
mask_prob_mc = [np.zeros((self.conf.val_batch_size, self.conf.height, self.conf.width, self.conf.num_cls))
for _ in range(self.conf.monte_carlo_simulations)]
feed_dict = {self.inputs_pl: data_x,
self.labels_pl: data_y,
self.is_training_pl: True,
self.with_dropout_pl: True,
self.keep_prob_pl: self.conf.keep_prob}
for mc_iter in range(self.conf.monte_carlo_simulations):
inputs, mask, mask_prob, mask_pred = self.sess.run([self.inputs_pl,
self.labels_pl,
self.y_prob,
self.y_pred], feed_dict=feed_dict)
mask_prob_mc[mc_iter] = mask_prob
mask_pred_mc[mc_iter] = mask_pred
prob_mean = np.nanmean(mask_prob_mc, axis=0)
prob_variance = np.var(mask_prob_mc, axis=0)
pred = np.argmax(prob_mean, axis=-1)
# var_one = np.nanmean(prob_variance, axis=-1)
# var_one = var_calculate_2d(pred, prob_variance)
var_one = predictive_entropy(prob_mean)
# var_one = mutual_info(prob_mean, mask_prob_mc)
hist += get_hist(pred.flatten(), mask.flatten(), num_cls=self.conf.num_cls)
# if all_inputs.shape[0] < 6:
# ii = np.random.randint(self.conf.val_batch_size)
# ii = 1
all_inputs = np.concatenate((all_inputs, inputs.reshape(-1, self.conf.height, self.conf.width,
self.conf.channel)), axis=0)
all_mask = np.concatenate((all_mask, mask.reshape(-1, self.conf.height, self.conf.width)), axis=0)
all_pred = np.concatenate((all_pred, pred.reshape(-1, self.conf.height, self.conf.width)), axis=0)
all_var = np.concatenate((all_var, var_one.reshape(-1, self.conf.height, self.conf.width)), axis=0)
cls_uncertainty = np.concatenate((cls_uncertainty,
prob_variance.reshape(-1, self.conf.height, self.conf.width,
self.conf.num_cls)),
axis=0)
# else:
self.visualize(all_inputs, all_mask, all_pred, all_var, cls_uncertainty, train_step=train_step,
mode='test')
import h5py
h5f = h5py.File('dropconnect_entropy_uncertainty.h5', 'w')
h5f.create_dataset('x', data=all_inputs)
h5f.create_dataset('y', data=all_mask)
h5f.create_dataset('y_pred', data=all_pred)
h5f.create_dataset('y_var', data=all_var)
h5f.create_dataset('cls_uncertainty', data=cls_uncertainty)
h5f.close()
# h5f = h5py.File('dropout_pred_uncertainty.h5', 'r')
# all_mask = h5f['y'][:]
# all_pred = h5f['y_pred'][:]
# all_var = h5f['y_var'][:]
# h5f.close()
# uncertainty_measure = get_uncertainty_precision(all_mask, all_pred, all_var)
# print('Uncertainty Quality Measure = {}'.format(uncertainty_measure))
# break
IOU, ACC = compute_iou(hist)
mean_IOU = np.mean(IOU)
print('****** IoU & ACC ******')
print('Mean IoU = {0:.01%}'.format(mean_IOU))
for ii in range(self.conf.num_cls):
print(' - {0} class: IoU={1:.01%}, ACC={2:.01%}'.format(self.conf.label_name[ii], IOU[ii], ACC[ii]))
print('-' * 20)