def evaluate(self, input_data, labels):
"""
Evaluate neural network by provided input data and labels/reconstruction target to get back a metrics score
:param input_data: Data to be inferred with neural network
:type input_data: ndarray
:param labels: labels
:type labels: ndarray
:return: metrics score
:rtype: float
:History: 2018-May-20 - Written - Henry Leung (University of Toronto)
"""
self.has_model_check()
input_data = list_to_dict(self.keras_model.input_names, input_data)
labels = list_to_dict(self.keras_model.output_names, labels)
# check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again)
if self.input_normalizer is None:
self.input_normalizer = Normalizer(mode=self.input_norm_mode)
self.labels_normalizer = Normalizer(mode=self.labels_norm_mode)
norm_data = self.input_normalizer.normalize(input_data)
self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels
norm_labels = self.labels_normalizer.normalize(labels)
self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels
else:
norm_data = self.input_normalizer.normalize(input_data, calc=False)
norm_labels = self.labels_normalizer.normalize(labels, calc=False)
norm_data = self._tensor_dict_sanitize(norm_data,
self.keras_model.input_names)
norm_labels = self._tensor_dict_sanitize(norm_labels,
self.keras_model.input_names)
total_num = input_data['input'].shape[0]
eval_batchsize = self.batch_size if total_num > self.batch_size else total_num
steps = total_num // self.batch_size if total_num > self.batch_size else 1
start_time = time.time()
print("Starting Evaluation")
evaluate_generator = CVAEDataGenerator(batch_size=eval_batchsize,
shuffle=False,
steps_per_epoch=steps,
data=[norm_data, norm_labels])
scores = self.keras_model.evaluate(evaluate_generator)
if isinstance(scores, float): # make sure scores is iterable
scores = list(str(scores))
outputname = self.keras_model.output_names
funcname = self.keras_model.metrics_names
print(
f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed'
)
return list_to_dict(funcname, scores)
def test(self, input_data):
"""
Use the neural network to do inference
:param input_data: Data to be inferred with neural network
:type input_data: ndarray
:return: prediction and prediction uncertainty
:rtype: ndarry
:History: 2017-Dec-06 - Written - Henry Leung (University of Toronto)
"""
self.has_model_check()
input_data = self.pre_testing_checklist_master(input_data)
input_array = self.input_normalizer.normalize(input_data, calc=False)
total_test_num = input_data['input'].shape[0] # Number of testing data
# for number of training data smaller than batch_size
if total_test_num < self.batch_size:
self.batch_size = total_test_num
# Due to the nature of how generator works, no overlapped prediction
data_gen_shape = (total_test_num // self.batch_size) * self.batch_size
remainder_shape = total_test_num - data_gen_shape # Remainder from generator
# TODO: named output????
predictions = np.zeros((total_test_num, self._labels_shape['output']))
norm_data_main = {}
norm_data_remainder = {}
for name in input_array.keys():
norm_data_main.update({name: input_array[name][:data_gen_shape]})
norm_data_remainder.update({name: input_array[name][data_gen_shape:]})
start_time = time.time()
print("Starting Inference")
# Data Generator for prediction
prediction_generator = CNNPredDataGenerator(batch_size=self.batch_size,
shuffle=False,
steps_per_epoch=total_test_num // self.batch_size,
data=[norm_data_main])
predictions[:data_gen_shape] = np.asarray(self.keras_model.predict_generator(prediction_generator))
if remainder_shape != 0:
# assume its caused by mono images, so need to expand dim by 1
for name in input_array.keys():
if len(norm_data_remainder[name][0].shape) != len(self._input_shape[name]):
norm_data_remainder.update({name: np.expand_dims(norm_data_remainder[name], axis=-1)})
result = self.keras_model.predict(norm_data_remainder)
predictions[data_gen_shape:] = result.reshape((remainder_shape, self._labels_shape['output']))
if self.labels_normalizer is not None:
predictions = self.labels_normalizer.denormalize(list_to_dict(self.keras_model.output_names, predictions))
else:
predictions *= self.labels_std
predictions += self.labels_mean
print(f'Completed Inference, {(time.time() - start_time):.{2}f}s elapsed')
return predictions['output']
def predict(self, input_data):
"""
Use the neural network to do inference
:param input_data: Data to be inferred with neural network
:type input_data: ndarray
:return: prediction and prediction uncertainty
:rtype: ndarry
:History: 2017-Dec-06 - Written - Henry Leung (University of Toronto)
"""
self.has_model_check()
input_data = self.pre_testing_checklist_master(input_data)
input_array = self.input_normalizer.normalize(input_data, calc=False)
total_test_num = input_data['input'].shape[0] # Number of testing data
# for number of training data smaller than batch_size
if total_test_num < self.batch_size:
self.batch_size = total_test_num
# Due to the nature of how generator works, no overlapped prediction
data_gen_shape = (total_test_num // self.batch_size) * self.batch_size
remainder_shape = total_test_num - data_gen_shape # Remainder from generator
# TODO: named output????
predictions = np.zeros((total_test_num, self._labels_shape['output']))
norm_data_main = {}
norm_data_remainder = {}
for name in input_array.keys():
norm_data_main.update({name: input_array[name][:data_gen_shape]})
norm_data_remainder.update({name: input_array[name][data_gen_shape:]})
norm_data_main = self._tensor_dict_sanitize(norm_data_main, self.keras_model.input_names)
norm_data_remainder = self._tensor_dict_sanitize(norm_data_remainder, self.keras_model.input_names)
# Data Generator for prediction
with tqdm(total=total_test_num, unit="sample") as pbar:
prediction_generator = CNNPredDataGenerator(batch_size=self.batch_size,
shuffle=False,
steps_per_epoch=total_test_num // self.batch_size,
data=[norm_data_main],
pbar=pbar)
predictions[:data_gen_shape] = np.asarray(self.keras_model.predict(prediction_generator))
if remainder_shape != 0:
remainder_generator = CNNPredDataGenerator(batch_size=remainder_shape,
shuffle=False,
steps_per_epoch=1,
data=[norm_data_remainder],
pbar=pbar)
predictions[data_gen_shape:] = np.asarray(self.keras_model.predict(remainder_generator))
if self.labels_normalizer is not None:
predictions = self.labels_normalizer.denormalize(list_to_dict(self.keras_model.output_names, predictions))
else:
predictions *= self.labels_std
predictions += self.labels_mean
return predictions['output']
def mode_checker(self, data):
if type(data) is not dict:
dict_flag = False
data = {"Temp": data}
self.mean_labels = {"Temp": self.mean_labels}
self.std_labels = {"Temp": self.std_labels}
else:
dict_flag = True
master_data = {}
if type(self.normalization_mode) is not dict:
self.normalization_mode = list_to_dict(
data.keys(), to_iterable(self.normalization_mode))
for name in data.keys(): # normalize data for each named inputs
if data[name].ndim == 1:
data_array = np.expand_dims(data[name], 1)
else:
data_array = np.array(data[name])
self.normalization_mode.update({
name:
str(self.normalization_mode[name])
}) # just to prevent unnecessary type issue
if data_array.dtype == bool:
if self.normalization_mode[
name] != '0': # binary classification case
warnings.warn(
"Data type is detected as bool, setting normalization_mode to 0 which is "
"doing nothing because no normalization can be done on bool"
)
self.normalization_mode[name] = '0'
data_array = data_array.astype(
np.float32,
copy=False) # need to convert data to float in every case
if self.normalization_mode[name] == '0':
self.featurewise_center.update({name: False})
self.datasetwise_center.update({name: False})
self.featurewise_stdalization.update({name: False})
self.datasetwise_stdalization.update({name: False})
self.mean_labels.update({name: np.array([0.])})
self.std_labels.update({name: np.array([1.])})
elif self.normalization_mode[name] == '1':
self.featurewise_center.update({name: False})
self.datasetwise_center.update({name: True})
self.featurewise_stdalization.update({name: False})
self.datasetwise_stdalization.update({name: True})
elif self.normalization_mode[name] == '2':
self.featurewise_center.update({name: True})
self.datasetwise_center.update({name: False})
self.featurewise_stdalization.update({name: True})
self.datasetwise_stdalization.update({name: False})
elif self.normalization_mode[name] == '3':
self.featurewise_center.update({name: True})
self.datasetwise_center.update({name: False})
self.featurewise_stdalization.update({name: False})
self.datasetwise_stdalization.update({name: False})
elif self.normalization_mode[
name] == '3s': # allow custom function, default to use sigmoid to normalize
self.featurewise_center.update({name: False})
self.datasetwise_center.update({name: False})
self.featurewise_stdalization.update({name: False})
self.datasetwise_stdalization.update({name: False})
if self._custom_norm_func is None:
self._custom_norm_func = sigmoid
if self._custom_denorm_func is None:
self._custom_denorm_func = sigmoid_inv
self.mean_labels.update({name: np.array([0.])})
self.std_labels.update({name: np.array([1.])})
elif self.normalization_mode[name] == '4':
self.featurewise_center.update({name: False})
self.datasetwise_center.update({name: False})
self.featurewise_stdalization.update({name: True})
self.datasetwise_stdalization.update({name: False})
elif self.normalization_mode[name] == '255':
# Used to normalize 8bit images
self.featurewise_center.update({name: False})
self.datasetwise_center.update({name: False})
self.featurewise_stdalization.update({name: False})
self.datasetwise_stdalization.update({name: False})
self.mean_labels.update({name: np.array([0.])})
self.std_labels.update({name: np.array([255.])})
else:
raise ValueError(
f"Unknown Mode -> {self.normalization_mode[name]}")
master_data.update({name: data_array})
return master_data, dict_flag
def load_folder(folder=None):
"""
To load astroNN model object from folder
:param folder: [optional] you should provide folder name if outside folder, do not specific when you are inside the folder
:type folder: str
:return: astroNN Neural Network instance
:rtype: astroNN.nn.NeuralNetMaster.NeuralNetMaster
:History: 2017-Dec-29 - Written - Henry Leung (University of Toronto)
"""
currentdir = os.getcwd()
if folder is not None:
fullfilepath = os.path.join(currentdir, folder)
else:
fullfilepath = currentdir
astronn_model_obj = None
if folder is not None and os.path.isfile(
os.path.join(folder, 'astroNN_model_parameter.json')) is True:
with open(os.path.join(folder, 'astroNN_model_parameter.json')) as f:
parameter = json.load(f)
f.close()
elif os.path.isfile('astroNN_model_parameter.json') is True:
with open('astroNN_model_parameter.json') as f:
parameter = json.load(f)
f.close()
elif folder is not None and not os.path.exists(folder):
raise IOError('Folder not exists: ' +
str(currentdir + os.sep + folder))
else:
raise FileNotFoundError(
'Are you sure this is an astroNN generated folder?')
identifier = parameter['id']
unknown_model_message = f'Unknown model identifier -> {identifier}!'
# need to point to the actual neural network if non-travial location
if identifier == 'Galaxy10CNN':
astronn_model_obj = Galaxy10CNN()
else:
# else try to import it from standard way
try:
astronn_model_obj = getattr(
importlib.import_module(f"astroNN.models"), identifier)()
except ImportError:
# try to load custom model from CUSTOM_MODEL_PATH if none are working
CUSTOM_MODEL_PATH = custom_model_path_reader()
# try the current folder and see if there is any .py on top of CUSTOM_MODEL_PATH
list_py_files = [
os.path.join(fullfilepath, f) for f in os.listdir(fullfilepath)
if f.endswith(".py")
]
if CUSTOM_MODEL_PATH is None and list_py_files is None:
print("\n")
raise TypeError(unknown_model_message)
else:
for path_list in (
path_list
for path_list in [CUSTOM_MODEL_PATH, list_py_files]
if path_list is not None):
for path in path_list:
head, tail = os.path.split(path)
sys.path.insert(0, head)
try:
model = getattr(
importlib.import_module(tail.strip('.py')),
str(identifier))
astronn_model_obj = model()
except AttributeError:
pass
if astronn_model_obj is None:
print("\n")
raise TypeError(unknown_model_message)
astronn_model_obj.currentdir = currentdir
astronn_model_obj.fullfilepath = fullfilepath
astronn_model_obj.folder_name = folder if folder is not None else os.path.basename(
os.path.normpath(currentdir))
# Must have parameter
astronn_model_obj._input_shape = parameter['input']
astronn_model_obj._labels_shape = parameter['labels']
if type(astronn_model_obj._input_shape) is not dict:
astronn_model_obj._input_shape = {
"input": astronn_model_obj._input_shape
}
if type(astronn_model_obj._labels_shape) is not dict:
astronn_model_obj._labels_shape = {
"output": astronn_model_obj._labels_shape
}
astronn_model_obj.num_hidden = parameter['hidden']
astronn_model_obj.input_norm_mode = parameter['input_norm_mode']
astronn_model_obj.labels_norm_mode = parameter['labels_norm_mode']
astronn_model_obj.batch_size = parameter['batch_size']
astronn_model_obj.targetname = parameter['targetname']
astronn_model_obj.val_size = parameter['valsize']
# Conditional parameter depends on neural net architecture
try:
astronn_model_obj.num_filters = parameter['filternum']
except KeyError:
pass
try:
astronn_model_obj.filter_len = parameter['filterlen']
except KeyError:
pass
try:
pool_length = parameter['pool_length']
if pool_length is not None:
if isinstance(pool_length, int): # multi-dimensional case
astronn_model_obj.pool_length = parameter['pool_length']
else:
astronn_model_obj.pool_length = list(parameter['pool_length'])
except KeyError or TypeError:
pass
try:
# need to convert to int because of keras do not want array or list
astronn_model_obj.latent_dim = int(parameter['latent'])
except KeyError:
pass
try:
astronn_model_obj.task = parameter['task']
except KeyError:
pass
try:
astronn_model_obj.dropout_rate = parameter['dropout_rate']
except KeyError:
pass
try:
# if inverse model precision exists, so does length_scale
astronn_model_obj.inv_model_precision = parameter['inv_tau']
astronn_model_obj.length_scale = parameter['length_scale']
except KeyError:
pass
try:
astronn_model_obj.l1 = parameter['l1']
except KeyError:
pass
try:
astronn_model_obj.l2 = parameter['l2']
except KeyError:
pass
try:
astronn_model_obj.maxnorm = parameter['maxnorm']
except KeyError:
pass
try:
astronn_model_obj.input_names = parameter['input_names']
except KeyError:
astronn_model_obj.input_names = ['input']
try:
astronn_model_obj.output_names = parameter['output_names']
except KeyError:
astronn_model_obj.output_names = ['output']
try:
astronn_model_obj._last_layer_activation = parameter[
'last_layer_activation']
except KeyError:
pass
try:
astronn_model_obj.activation = parameter['activation']
except KeyError:
pass
with h5py.File(os.path.join(astronn_model_obj.fullfilepath,
'model_weights.h5'),
mode='r') as f:
training_config = json.loads(f.attrs['training_config'])
optimizer_config = training_config['optimizer_config']
optimizer = optimizers.deserialize(optimizer_config)
model_config = json.loads(f.attrs['model_config'])
# input/name names, mean, std
input_names = []
output_names = []
for lay in model_config["config"]["input_layers"]:
input_names.append(lay[0])
for lay in model_config["config"]["output_layers"]:
output_names.append(lay[0])
astronn_model_obj.input_mean = list_to_dict(
input_names, dict_list_to_dict_np(parameter['input_mean']))
astronn_model_obj.labels_mean = list_to_dict(
output_names, dict_list_to_dict_np(parameter['labels_mean']))
astronn_model_obj.input_std = list_to_dict(
input_names, dict_list_to_dict_np(parameter['input_std']))
astronn_model_obj.labels_std = list_to_dict(
output_names, dict_list_to_dict_np(parameter['labels_std']))
# Recover loss functions and metrics.
losses_raw = convert_custom_objects(training_config['loss'])
try:
try:
loss = [
losses_lookup(losses_raw[_loss]) for _loss in losses_raw
]
except TypeError:
loss = losses_lookup(losses_raw)
except:
raise LookupError("Cant lookup loss")
metrics_raw = convert_custom_objects(training_config['metrics'])
# its weird that keras needs -> metrics[metric][0] instead of metrics[metric] likes losses
try:
try:
if version.parse(tf.__version__) >= version.parse("2.4.0"):
metrics = [
losses_lookup(_metric['config']['fn'])
for _metric in metrics_raw[0]
]
else:
metrics = [
losses_lookup(metrics_raw[_metric][0])
for _metric in metrics_raw
]
except TypeError:
metrics = [losses_lookup(metrics_raw[0])]
except:
metrics = metrics_raw
sample_weight_mode = training_config['sample_weight_mode'] if \
hasattr(training_config, 'sample_weight_mode') else None
loss_weights = training_config['loss_weights']
weighted_metrics = None
# compile the model
astronn_model_obj.compile(optimizer=optimizer,
loss=loss,
metrics=metrics,
weighted_metrics=weighted_metrics,
loss_weights=loss_weights,
sample_weight_mode=sample_weight_mode)
# set weights
astronn_model_obj.keras_model.load_weights(
os.path.join(astronn_model_obj.fullfilepath, 'model_weights.h5'))
# Build train function (to get weight updates), need to consider Sequential model too
astronn_model_obj.keras_model.make_train_function()
try:
optimizer_weights_group = f['optimizer_weights']
if version.parse(h5py.__version__) >= version.parse("3.0"):
optimizer_weight_names = [
n for n in optimizer_weights_group.attrs['weight_names']
]
else:
optimizer_weight_names = [
n.decode('utf8')
for n in optimizer_weights_group.attrs['weight_names']
]
optimizer_weight_values = [
optimizer_weights_group[n] for n in optimizer_weight_names
]
astronn_model_obj.keras_model.optimizer._create_all_weights(
astronn_model_obj.keras_model.trainable_variables)
astronn_model_obj.keras_model.optimizer.set_weights(
optimizer_weight_values)
except KeyError:
warnings.warn(
"Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer."
)
# create normalizer and set correct mean and std
astronn_model_obj.input_normalizer = Normalizer(
mode=astronn_model_obj.input_norm_mode)
astronn_model_obj.labels_normalizer = Normalizer(
mode=astronn_model_obj.labels_norm_mode)
astronn_model_obj.input_normalizer.mean_labels = astronn_model_obj.input_mean
astronn_model_obj.input_normalizer.std_labels = astronn_model_obj.input_std
astronn_model_obj.labels_normalizer.mean_labels = astronn_model_obj.labels_mean
astronn_model_obj.labels_normalizer.std_labels = astronn_model_obj.labels_std
print("========================================================")
print(
f"Loaded astroNN model, model type: {astronn_model_obj.name} -> {identifier}"
)
print("========================================================")
return astronn_model_obj
def evaluate(self, input_data, labels, inputs_err=None, labels_err=None):
"""
Evaluate neural network by provided input data and labels and get back a metrics score
:param input_data: Data to be trained with neural network
:type input_data: ndarray
:param labels: Labels to be trained with neural network
:type labels: ndarray
:param inputs_err: Error for input_data (if any), same shape with input_data.
:type inputs_err: Union([NoneType, ndarray])
:param labels_err: Labels error (if any)
:type labels_err: Union([NoneType, ndarray])
:return: metrics score dictionary
:rtype: dict
:History: 2018-May-20 - Written - Henry Leung (University of Toronto)
"""
self.has_model_check()
if inputs_err is None:
inputs_err = np.zeros_like(input_data)
if labels_err is None:
labels_err = np.zeros_like(labels)
input_data = {"input": input_data}
labels = {"output": labels}
# check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again)
if self.input_normalizer is None:
self.input_normalizer = Normalizer(mode=self.input_norm_mode)
self.labels_normalizer = Normalizer(mode=self.labels_norm_mode)
norm_data = self.input_normalizer.normalize(input_data)
self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels
norm_labels = self.labels_normalizer.normalize(labels)
self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels
else:
norm_data = self.input_normalizer.normalize(input_data, calc=False)
norm_labels = self.labels_normalizer.normalize(labels, calc=False)
# No need to care about Magic number as loss function looks for magic num in y_true only
norm_input_err = inputs_err / self.input_std['input']
norm_labels_err = labels_err / self.labels_std['output']
norm_data.update({
"input_err": norm_input_err,
"labels_err": norm_labels_err
})
norm_labels.update({
"labels_err": norm_labels_err,
"variance_output": norm_labels["output"]
})
total_num = input_data['input'].shape[0]
eval_batchsize = self.batch_size if total_num > self.batch_size else total_num
steps = total_num // self.batch_size if total_num > self.batch_size else 1
start_time = time.time()
print("Starting Evaluation")
evaluate_generator = BayesianCNNDataGenerator(
batch_size=eval_batchsize,
shuffle=False,
steps_per_epoch=steps,
data=[norm_data, norm_labels])
scores = self.keras_model.evaluate_generator(evaluate_generator)
if isinstance(scores, float): # make sure scores is iterable
scores = list(str(scores))
outputname = self.keras_model.output_names
funcname = self.keras_model.metrics_names
print(
f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed'
)
return list_to_dict(funcname, scores)
def test(self, input_data, inputs_err=None):
"""
Test model, High performance version designed for fast variational inference on GPU
:param input_data: Data to be inferred with neural network
:type input_data: ndarray
:param inputs_err: Error for input_data, same shape with input_data.
:type inputs_err: Union([NoneType, ndarray])
:return: prediction and prediction uncertainty
:History:
| 2018-Jan-06 - Written - Henry Leung (University of Toronto)
| 2018-Apr-12 - Updated - Henry Leung (University of Toronto)
"""
self.has_model_check()
if gpu_availability() is False and self.mc_num > 25:
warnings.warn(
f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can '
f'potentially be very slow! \n '
f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n'
f'This is just a warning, and will not shown if mc_num < 25 on CPU'
)
if self.mc_num < 2:
raise AttributeError("mc_num cannot be smaller than 2")
# if no error array then just zeros
if inputs_err is None:
inputs_err = np.zeros_like(input_data)
else:
inputs_err = np.atleast_2d(inputs_err)
inputs_err /= self.input_std['input']
input_data = {"input": input_data, "input_err": inputs_err}
input_data = self.pre_testing_checklist_master(input_data)
if self.input_normalizer is not None:
input_array = self.input_normalizer.normalize(input_data,
calc=False)
else:
# Prevent shallow copy issue
input_array = np.array(input_data)
input_array -= self.input_mean['input']
input_array /= self.input_std['input']
total_test_num = input_data['input'].shape[0] # Number of testing data
# for number of training data smaller than batch_size
if total_test_num < self.batch_size:
batch_size = total_test_num
else:
batch_size = self.batch_size
# Due to the nature of how generator works, no overlapped prediction
data_gen_shape = (total_test_num // batch_size) * batch_size
remainder_shape = total_test_num - data_gen_shape # Remainder from generator
norm_data_main = {}
norm_data_remainder = {}
for name in input_array.keys():
norm_data_main.update({name: input_array[name][:data_gen_shape]})
norm_data_remainder.update(
{name: input_array[name][data_gen_shape:]})
start_time = time.time()
print("Starting Dropout Variational Inference")
# Data Generator for prediction
prediction_generator = BayesianCNNPredDataGenerator(
batch_size=batch_size,
shuffle=False,
steps_per_epoch=data_gen_shape // batch_size,
data=[norm_data_main])
new = FastMCInference(self.mc_num)(self.keras_model_predict)
result = np.asarray(new.predict_generator(prediction_generator))
if remainder_shape != 0: # deal with remainder
remainder_generator = BayesianCNNPredDataGenerator(
batch_size=remainder_shape,
shuffle=False,
steps_per_epoch=1,
data=[norm_data_remainder])
remainder_result = np.asarray(
new.predict_generator(remainder_generator))
if remainder_shape == 1:
remainder_result = np.expand_dims(remainder_result, axis=0)
result = np.concatenate((result, remainder_result))
# in case only 1 test data point, in such case we need to add a dimension
if result.ndim < 3 and batch_size == 1:
result = np.expand_dims(result, axis=0)
half_first_dim = result.shape[
1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong
predictions = result[:, :half_first_dim, 0] # mean prediction
mc_dropout_uncertainty = result[:, :half_first_dim, 1] * (
self.labels_std['output']**2) # model uncertainty
predictions_var = np.exp(result[:, half_first_dim:, 0]) * (
self.labels_std['output']**2) # predictive uncertainty
print(
f'Completed Dropout Variational Inference with {self.mc_num} forward passes, '
f'{(time.time() - start_time):.{2}f}s elapsed')
if self.labels_normalizer is not None:
predictions = self.labels_normalizer.denormalize(
list_to_dict([self.keras_model.output_names[0]], predictions))
predictions = predictions['output']
else:
predictions *= self.labels_std['output']
predictions += self.labels_mean['output']
if self.task == 'regression':
# Predictive variance
pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty
pred_uncertainty = np.sqrt(pred_var) # Convert back to std error
# final correction from variance to standard derivation
mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty)
predictive_uncertainty = np.sqrt(predictions_var)
elif self.task == 'classification':
# we want entropy for classification uncertainty
predicted_class = np.argmax(predictions, axis=1)
mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float)
predictive_uncertainty = np.ones_like(predicted_class, dtype=float)
# center variance
predictions_var -= 1.
for i in range(predicted_class.shape[0]):
all_prediction = np.array(predictions[i, :])
mc_dropout_uncertainty[i] = -np.sum(
all_prediction * np.log(all_prediction))
predictive_uncertainty[i] = predictions_var[i,
predicted_class[i]]
pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty
# We only want the predicted class back
predictions = predicted_class
elif self.task == 'binary_classification':
# we want entropy for classification uncertainty, so need prediction in logits space
mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions),
axis=0)
# need to activate before round to int so that the prediction is always 0 or 1
predictions = np.rint(sigmoid(predictions))
predictive_uncertainty = predictions_var
pred_uncertainty = mc_dropout_uncertainty + predictions_var
else:
raise AttributeError('Unknown Task')
return predictions, {
'total': pred_uncertainty,
'model': mc_dropout_uncertainty,
'predictive': predictive_uncertainty
}
def predict(self, input_data):
"""
Use the neural network to do inference and get reconstructed data
:param input_data: Data to be inferred with neural network
:type input_data: ndarray
:return: reconstructed data
:rtype: ndarry
:History: 2017-Dec-06 - Written - Henry Leung (University of Toronto)
"""
input_data = self.pre_testing_checklist_master(input_data)
if self.input_normalizer is not None:
input_array = self.input_normalizer.normalize(input_data, calc=False)
else:
# Prevent shallow copy issue
input_array = np.array(input_data)
input_array -= self.input_mean
input_array /= self.input_std
total_test_num = input_data['input'].shape[0] # Number of testing data
# for number of training data smaller than batch_size
if total_test_num < self.batch_size:
self.batch_size = total_test_num
# Due to the nature of how generator works, no overlapped prediction
data_gen_shape = (total_test_num // self.batch_size) * self.batch_size
remainder_shape = total_test_num - data_gen_shape # Remainder from generator
predictions = np.zeros((total_test_num, self._labels_shape['output'], 1))
norm_data_main = {}
norm_data_remainder = {}
for name in input_array.keys():
norm_data_main.update({name: input_array[name][:data_gen_shape]})
norm_data_remainder.update({name: input_array[name][data_gen_shape:]})
norm_data_main = self._tensor_dict_sanitize(norm_data_main, self.keras_model.input_names)
norm_data_remainder = self._tensor_dict_sanitize(norm_data_remainder, self.keras_model.input_names)
start_time = time.time()
print("Starting Inference")
# Data Generator for prediction
with tqdm(total=total_test_num, unit="sample") as pbar:
prediction_generator = CVAEPredDataGenerator(batch_size=self.batch_size,
shuffle=False,
steps_per_epoch=total_test_num // self.batch_size,
data=[norm_data_main])
result = np.asarray(self.keras_model.predict(prediction_generator))
if remainder_shape != 0:
remainder_generator = CVAEPredDataGenerator(batch_size=remainder_shape,
shuffle=False,
steps_per_epoch=1,
data=[norm_data_remainder],
pbar=pbar)
remainder_result = np.asarray(self.keras_model.predict(remainder_generator))
result = np.concatenate((result, remainder_result))
if self.labels_normalizer is not None:
# TODO: handle named output in the future
predictions[:, :, 0] = self.labels_normalizer.denormalize(list_to_dict(self.keras_model.output_names,
predictions[:, :, 0]))['output']
else:
predictions[:, :, 0] *= self.labels_std
predictions[:, :, 0] += self.labels_mean
print(f'Completed Inference, {(time.time() - start_time):.{2}f}s elapsed')
return predictions
def jacobian_old(self, x=None, mean_output=False, denormalize=False):
"""
| Calculate jacobian of gradient of output to input
|
| Please notice that the de-normalize (if True) assumes the output depends on the input data first orderly
| in which the equation is simply jacobian divided the input scaling
:param x: Input Data
:type x: ndarray
:param mean_output: False to get all jacobian, True to get the mean
:type mean_output: boolean
:param denormalize: De-normalize Jacobian
:type denormalize: bool
:History: 2017-Nov-20 - Written - Henry Leung (University of Toronto)
"""
self.has_model_check()
if x is None:
raise ValueError('Please provide data to calculate the jacobian')
x = list_to_dict(self.keras_model.input_names, x)
if self.input_normalizer is not None:
x_data = self.input_normalizer.normalize(x, calc=False)
x_data = x_data['input']
else:
# Prevent shallow copy issue
x_data = np.array(x)
x_data -= self.input_mean
x_data /= self.input_std
try:
input_tens = self.keras_model_predict.get_layer("input").input
output_tens = self.keras_model_predict.get_layer("output").output
input_shape_expectation = self.keras_model_predict.get_layer(
"input").input_shape
except AttributeError:
input_tens = self.keras_model.get_layer("input").input
output_tens = self.keras_model.get_layer("output").output
input_shape_expectation = self.keras_model.get_layer(
"input").input_shape
start_time = time.time()
if len(input_shape_expectation) == 1:
input_shape_expectation = input_shape_expectation[0]
if len(input_shape_expectation) == 3:
x_data = np.atleast_3d(x_data)
grad_list = []
for j in range(self._labels_shape['output']):
grad_list.append(tf.gradients(output_tens[0, j], input_tens))
final_stack = tf.stack(tf.squeeze(grad_list))
jacobian = np.ones((x_data.shape[0], self._labels_shape['output'],
x_data.shape[1]),
dtype=np.float32)
for i in range(x_data.shape[0]):
x_in = x_data[i:i + 1]
jacobian[i, :, :] = get_session().run(
final_stack,
feed_dict={
input_tens: x_in,
tfk.backend.learning_phase(): 0
})
elif len(input_shape_expectation) == 4:
monoflag = False
if len(x_data.shape) < 4:
monoflag = True
x_data = x_data[:, :, :, np.newaxis]
jacobian = np.ones(
(x_data.shape[0], self._labels_shape['output'],
x_data.shape[1], x_data.shape[2], x_data.shape[3]),
dtype=np.float32)
grad_list = []
for j in range(self._labels_shape['output']):
grad_list.append(
tf.gradients(
self.keras_model.get_layer("output").output[0, j],
input_tens))
final_stack = tf.stack(tf.squeeze(grad_list))
for i in range(x_data.shape[0]):
x_in = x_data[i:i + 1]
if monoflag is False:
jacobian[i, :, :, :, :] = get_session().run(
final_stack,
feed_dict={
input_tens: x_in,
tfk.backend.learning_phase(): 0
})
else:
jacobian[i, :, :, :, 0] = get_session().run(
final_stack,
feed_dict={
input_tens: x_in,
tfk.backend.learning_phase(): 0
})
else:
raise ValueError(
'Input data shape do not match neural network expectation')
if mean_output is True:
jacobian_master = np.mean(jacobian, axis=0)
else:
jacobian_master = np.array(jacobian)
jacobian_master = np.squeeze(jacobian_master)
if denormalize:
if self.input_std is not None:
jacobian_master = jacobian_master / self.input_std
if self.labels_std is not None:
try:
jacobian_master = jacobian_master * self.labels_std
except ValueError:
jacobian_master = jacobian_master * self.labels_std.reshape(
-1, 1)
print(
f'Finished gradient calculation, {(time.time() - start_time):.{2}f} seconds elapsed'
)
return jacobian_master
def predict_dataset(self, file):
class BayesianCNNPredDataGeneratorV2(GeneratorMaster):
def __init__(self,
batch_size,
shuffle,
steps_per_epoch,
manual_reset=False,
pbar=None,
nn_model=None):
super().__init__(batch_size=batch_size,
shuffle=shuffle,
steps_per_epoch=steps_per_epoch,
data=None,
manual_reset=manual_reset)
self.pbar = pbar
# initial idx
self.idx_list = self._get_exploration_order(range(len(file)))
self.current_idx = 0
self.nn_model = nn_model
def _data_generation(self, idx_list_temp):
# Generate data
inputs = self.nn_model.input_normalizer.normalize(
{
"input": file[idx_list_temp],
"input_err": np.zeros_like(file[idx_list_temp])
},
calc=False)
x = self.input_d_checking(inputs,
np.arange(len(idx_list_temp)))
return x
def __getitem__(self, index):
x = self._data_generation(
self.idx_list[index * self.batch_size:(index + 1) *
self.batch_size])
if self.pbar: self.pbar.update(self.batch_size)
return x
def on_epoch_end(self):
# shuffle the list when epoch ends for the next epoch
self.idx_list = self._get_exploration_order(range(len(file)))
self.has_model_check()
if gpu_availability() is False and self.mc_num > 25:
warnings.warn(
f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can '
f'potentially be very slow! \n '
f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n'
f'This is just a warning, and will not shown if mc_num < 25 on CPU'
)
if self.mc_num < 2:
raise AttributeError("mc_num cannot be smaller than 2")
total_test_num = len(file) # Number of testing data
# for number of training data smaller than batch_size
if total_test_num < self.batch_size:
batch_size = total_test_num
else:
batch_size = self.batch_size
# Due to the nature of how generator works, no overlapped prediction
data_gen_shape = (total_test_num // batch_size) * batch_size
remainder_shape = total_test_num - data_gen_shape # Remainder from generator
# Data Generator for prediction
with tqdm(total=total_test_num, unit="sample") as pbar:
pbar.set_postfix({'Monte-Carlo': self.mc_num})
# suppress pfor warning from TF
old_level = tf.get_logger().level
tf.get_logger().setLevel('ERROR')
prediction_generator = BayesianCNNPredDataGeneratorV2(
batch_size=batch_size,
shuffle=False,
steps_per_epoch=data_gen_shape // batch_size,
pbar=pbar,
nn_model=self)
new = FastMCInference(self.mc_num)(self.keras_model_predict)
result = np.asarray(new.predict(prediction_generator))
if remainder_shape != 0: # deal with remainder
remainder_generator = BayesianCNNPredDataGeneratorV2(
batch_size=remainder_shape,
shuffle=False,
steps_per_epoch=1,
pbar=pbar,
nn_model=self)
remainder_result = np.asarray(new.predict(remainder_generator))
if remainder_shape == 1:
remainder_result = np.expand_dims(remainder_result, axis=0)
result = np.concatenate((result, remainder_result))
tf.get_logger().setLevel(old_level)
# in case only 1 test data point, in such case we need to add a dimension
if result.ndim < 3 and batch_size == 1:
result = np.expand_dims(result, axis=0)
half_first_dim = result.shape[
1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong
predictions = result[:, :half_first_dim, 0] # mean prediction
mc_dropout_uncertainty = result[:, :half_first_dim, 1] * (
self.labels_std['output']**2) # model uncertainty
predictions_var = np.exp(result[:, half_first_dim:, 0]) * (
self.labels_std['output']**2) # predictive uncertainty
if self.labels_normalizer is not None:
predictions = self.labels_normalizer.denormalize(
list_to_dict([self.keras_model.output_names[0]], predictions))
predictions = predictions['output']
else:
predictions *= self.labels_std['output']
predictions += self.labels_mean['output']
if self.task == 'regression':
# Predictive variance
pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty
pred_uncertainty = np.sqrt(pred_var) # Convert back to std error
# final correction from variance to standard derivation
mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty)
predictive_uncertainty = np.sqrt(predictions_var)
elif self.task == 'classification':
# we want entropy for classification uncertainty
predicted_class = np.argmax(predictions, axis=1)
mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float)
predictive_uncertainty = np.ones_like(predicted_class, dtype=float)
# center variance
predictions_var -= 1.
for i in range(predicted_class.shape[0]):
all_prediction = np.array(predictions[i, :])
mc_dropout_uncertainty[i] = -np.sum(
all_prediction * np.log(all_prediction))
predictive_uncertainty[i] = predictions_var[i,
predicted_class[i]]
pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty
# We only want the predicted class back
predictions = predicted_class
elif self.task == 'binary_classification':
# we want entropy for classification uncertainty, so need prediction in logits space
mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions),
axis=0)
# need to activate before round to int so that the prediction is always 0 or 1
predictions = np.rint(sigmoid(predictions))
predictive_uncertainty = predictions_var
pred_uncertainty = mc_dropout_uncertainty + predictions_var
else:
raise AttributeError('Unknown Task')
return predictions, {
'total': pred_uncertainty,
'model': mc_dropout_uncertainty,
'predictive': predictive_uncertainty
}
