def test(self, input_data, inputs_err=None):
"""
Test model, High performance version designed for fast variational inference on GPU
:param input_data: Data to be inferred with neural network
:type input_data: ndarray
:param inputs_err: Error for input_data, same shape with input_data.
:type inputs_err: Union([NoneType, ndarray])
:return: prediction and prediction uncertainty
:History:
| 2018-Jan-06 - Written - Henry Leung (University of Toronto)
| 2018-Apr-12 - Updated - Henry Leung (University of Toronto)
"""
self.has_model_check()
if gpu_availability() is False and self.mc_num > 25:
warnings.warn(
f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can '
f'potentially be very slow! \n '
f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n'
f'This is just a warning, and will not shown if mc_num < 25 on CPU'
)
if self.mc_num < 2:
raise AttributeError("mc_num cannot be smaller than 2")
self.pre_testing_checklist_master()
input_data = np.atleast_2d(input_data)
if self.input_normalizer is not None:
input_array = self.input_normalizer.normalize(input_data,
calc=False)
else:
# Prevent shallow copy issue
input_array = np.array(input_data)
input_array -= self.input_mean
input_array /= self.input_std
# if no error array then just zeros
if inputs_err is None:
inputs_err = np.zeros_like(input_data)
else:
inputs_err = np.atleast_2d(inputs_err)
inputs_err /= self.input_std
total_test_num = input_data.shape[0] # Number of testing data
# for number of training data smaller than batch_size
if total_test_num < self.batch_size:
batch_size = total_test_num
else:
batch_size = self.batch_size
# Due to the nature of how generator works, no overlapped prediction
data_gen_shape = (total_test_num // batch_size) * batch_size
remainder_shape = total_test_num - data_gen_shape # Remainder from generator
start_time = time.time()
print("Starting Dropout Variational Inference")
# Data Generator for prediction
prediction_generator = BayesianCNNPredDataGenerator(
batch_size=batch_size,
shuffle=False,
steps_per_epoch=data_gen_shape // batch_size,
data=[input_array[:data_gen_shape], inputs_err[:data_gen_shape]])
new = FastMCInference(self.mc_num)(self.keras_model_predict)
result = np.asarray(new.predict_generator(prediction_generator))
if remainder_shape != 0: # deal with remainder
remainder_generator = BayesianCNNPredDataGenerator(
batch_size=remainder_shape,
shuffle=False,
steps_per_epoch=1,
data=[
input_array[data_gen_shape:], inputs_err[data_gen_shape:]
])
remainder_result = np.asarray(
new.predict_generator(remainder_generator))
if remainder_shape == 1:
remainder_result = np.expand_dims(remainder_result, axis=0)
result = np.concatenate((result, remainder_result))
# in case only 1 test data point, in such case we need to add a dimension
if result.ndim < 3 and batch_size == 1:
result = np.expand_dims(result, axis=0)
half_first_dim = result.shape[
1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong
predictions = result[:, :half_first_dim, 0] # mean prediction
mc_dropout_uncertainty = result[:, :half_first_dim, 1] * (
self.labels_std**2) # model uncertainty
predictions_var = np.exp(result[:, half_first_dim:, 0]) * (
self.labels_std**2) # predictive uncertainty
print(
f'Completed Dropout Variational Inference with {self.mc_num} forward passes, '
f'{(time.time() - start_time):.{2}f}s elapsed')
if self.labels_normalizer is not None:
predictions = self.labels_normalizer.denormalize(predictions)
else:
predictions *= self.labels_std
predictions += self.labels_mean
if self.task == 'regression':
# Predictive variance
pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty
pred_uncertainty = np.sqrt(pred_var) # Convert back to std error
# final correction from variance to standard derivation
mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty)
predictive_uncertainty = np.sqrt(predictions_var)
elif self.task == 'classification':
# we want entropy for classification uncertainty
predicted_class = np.argmax(predictions, axis=1)
mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float)
predictive_uncertainty = np.ones_like(predicted_class, dtype=float)
# center variance
predictions_var -= 1.
for i in range(predicted_class.shape[0]):
all_prediction = np.array(predictions[i, :])
mc_dropout_uncertainty[i] = -np.sum(
all_prediction * np.log(all_prediction))
predictive_uncertainty[i] = predictions_var[i,
predicted_class[i]]
pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty
# We only want the predicted class back
predictions = predicted_class
elif self.task == 'binary_classification':
# we want entropy for classification uncertainty, so need prediction in logits space
mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions),
axis=0)
# need to activate before round to int so that the prediction is always 0 or 1
predictions = np.rint(sigmoid(predictions))
predictive_uncertainty = predictions_var
pred_uncertainty = mc_dropout_uncertainty + predictions_var
else:
raise AttributeError('Unknown Task')
return predictions, {
'total': pred_uncertainty,
'model': mc_dropout_uncertainty,
'predictive': predictive_uncertainty
}
def predict_dataset(self, file):
class BayesianCNNPredDataGeneratorV2(GeneratorMaster):
def __init__(self,
batch_size,
shuffle,
steps_per_epoch,
manual_reset=False,
pbar=None,
nn_model=None):
super().__init__(batch_size=batch_size,
shuffle=shuffle,
steps_per_epoch=steps_per_epoch,
data=None,
manual_reset=manual_reset)
self.pbar = pbar
# initial idx
self.idx_list = self._get_exploration_order(range(len(file)))
self.current_idx = 0
self.nn_model = nn_model
def _data_generation(self, idx_list_temp):
# Generate data
inputs = self.nn_model.input_normalizer.normalize(
{
"input": file[idx_list_temp],
"input_err": np.zeros_like(file[idx_list_temp])
},
calc=False)
x = self.input_d_checking(inputs,
np.arange(len(idx_list_temp)))
return x
def __getitem__(self, index):
x = self._data_generation(
self.idx_list[index * self.batch_size:(index + 1) *
self.batch_size])
if self.pbar: self.pbar.update(self.batch_size)
return x
def on_epoch_end(self):
# shuffle the list when epoch ends for the next epoch
self.idx_list = self._get_exploration_order(range(len(file)))
self.has_model_check()
if gpu_availability() is False and self.mc_num > 25:
warnings.warn(
f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can '
f'potentially be very slow! \n '
f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n'
f'This is just a warning, and will not shown if mc_num < 25 on CPU'
)
if self.mc_num < 2:
raise AttributeError("mc_num cannot be smaller than 2")
total_test_num = len(file) # Number of testing data
# for number of training data smaller than batch_size
if total_test_num < self.batch_size:
batch_size = total_test_num
else:
batch_size = self.batch_size
# Due to the nature of how generator works, no overlapped prediction
data_gen_shape = (total_test_num // batch_size) * batch_size
remainder_shape = total_test_num - data_gen_shape # Remainder from generator
# Data Generator for prediction
with tqdm(total=total_test_num, unit="sample") as pbar:
pbar.set_postfix({'Monte-Carlo': self.mc_num})
# suppress pfor warning from TF
old_level = tf.get_logger().level
tf.get_logger().setLevel('ERROR')
prediction_generator = BayesianCNNPredDataGeneratorV2(
batch_size=batch_size,
shuffle=False,
steps_per_epoch=data_gen_shape // batch_size,
pbar=pbar,
nn_model=self)
new = FastMCInference(self.mc_num)(self.keras_model_predict)
result = np.asarray(new.predict(prediction_generator))
if remainder_shape != 0: # deal with remainder
remainder_generator = BayesianCNNPredDataGeneratorV2(
batch_size=remainder_shape,
shuffle=False,
steps_per_epoch=1,
pbar=pbar,
nn_model=self)
remainder_result = np.asarray(new.predict(remainder_generator))
if remainder_shape == 1:
remainder_result = np.expand_dims(remainder_result, axis=0)
result = np.concatenate((result, remainder_result))
tf.get_logger().setLevel(old_level)
# in case only 1 test data point, in such case we need to add a dimension
if result.ndim < 3 and batch_size == 1:
result = np.expand_dims(result, axis=0)
half_first_dim = result.shape[
1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong
predictions = result[:, :half_first_dim, 0] # mean prediction
mc_dropout_uncertainty = result[:, :half_first_dim, 1] * (
self.labels_std['output']**2) # model uncertainty
predictions_var = np.exp(result[:, half_first_dim:, 0]) * (
self.labels_std['output']**2) # predictive uncertainty
if self.labels_normalizer is not None:
predictions = self.labels_normalizer.denormalize(
list_to_dict([self.keras_model.output_names[0]], predictions))
predictions = predictions['output']
else:
predictions *= self.labels_std['output']
predictions += self.labels_mean['output']
if self.task == 'regression':
# Predictive variance
pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty
pred_uncertainty = np.sqrt(pred_var) # Convert back to std error
# final correction from variance to standard derivation
mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty)
predictive_uncertainty = np.sqrt(predictions_var)
elif self.task == 'classification':
# we want entropy for classification uncertainty
predicted_class = np.argmax(predictions, axis=1)
mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float)
predictive_uncertainty = np.ones_like(predicted_class, dtype=float)
# center variance
predictions_var -= 1.
for i in range(predicted_class.shape[0]):
all_prediction = np.array(predictions[i, :])
mc_dropout_uncertainty[i] = -np.sum(
all_prediction * np.log(all_prediction))
predictive_uncertainty[i] = predictions_var[i,
predicted_class[i]]
pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty
# We only want the predicted class back
predictions = predicted_class
elif self.task == 'binary_classification':
# we want entropy for classification uncertainty, so need prediction in logits space
mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions),
axis=0)
# need to activate before round to int so that the prediction is always 0 or 1
predictions = np.rint(sigmoid(predictions))
predictive_uncertainty = predictions_var
pred_uncertainty = mc_dropout_uncertainty + predictions_var
else:
raise AttributeError('Unknown Task')
return predictions, {
'total': pred_uncertainty,
'model': mc_dropout_uncertainty,
'predictive': predictive_uncertainty
}
