def evaluate(self, input_data, labels): """ Evaluate neural network by provided input data and labels/reconstruction target to get back a metrics score :param input_data: Data to be inferred with neural network :type input_data: ndarray :param labels: labels :type labels: ndarray :return: metrics score :rtype: float :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() input_data = list_to_dict(self.keras_model.input_names, input_data) labels = list_to_dict(self.keras_model.output_names, labels) # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) norm_data = self._tensor_dict_sanitize(norm_data, self.keras_model.input_names) norm_labels = self._tensor_dict_sanitize(norm_labels, self.keras_model.input_names) total_num = input_data['input'].shape[0] eval_batchsize = self.batch_size if total_num > self.batch_size else total_num steps = total_num // self.batch_size if total_num > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") evaluate_generator = CVAEDataGenerator(batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels]) scores = self.keras_model.evaluate(evaluate_generator) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = self.keras_model.metrics_names print( f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed' ) return list_to_dict(funcname, scores)
def test(self, input_data): """ Use the neural network to do inference :param input_data: Data to be inferred with neural network :type input_data: ndarray :return: prediction and prediction uncertainty :rtype: ndarry :History: 2017-Dec-06 - Written - Henry Leung (University of Toronto) """ self.has_model_check() input_data = self.pre_testing_checklist_master(input_data) input_array = self.input_normalizer.normalize(input_data, calc=False) total_test_num = input_data['input'].shape[0] # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: self.batch_size = total_test_num # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // self.batch_size) * self.batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator # TODO: named output???? predictions = np.zeros((total_test_num, self._labels_shape['output'])) norm_data_main = {} norm_data_remainder = {} for name in input_array.keys(): norm_data_main.update({name: input_array[name][:data_gen_shape]}) norm_data_remainder.update({name: input_array[name][data_gen_shape:]}) start_time = time.time() print("Starting Inference") # Data Generator for prediction prediction_generator = CNNPredDataGenerator(batch_size=self.batch_size, shuffle=False, steps_per_epoch=total_test_num // self.batch_size, data=[norm_data_main]) predictions[:data_gen_shape] = np.asarray(self.keras_model.predict_generator(prediction_generator)) if remainder_shape != 0: # assume its caused by mono images, so need to expand dim by 1 for name in input_array.keys(): if len(norm_data_remainder[name][0].shape) != len(self._input_shape[name]): norm_data_remainder.update({name: np.expand_dims(norm_data_remainder[name], axis=-1)}) result = self.keras_model.predict(norm_data_remainder) predictions[data_gen_shape:] = result.reshape((remainder_shape, self._labels_shape['output'])) if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize(list_to_dict(self.keras_model.output_names, predictions)) else: predictions *= self.labels_std predictions += self.labels_mean print(f'Completed Inference, {(time.time() - start_time):.{2}f}s elapsed') return predictions['output']
def predict(self, input_data): """ Use the neural network to do inference :param input_data: Data to be inferred with neural network :type input_data: ndarray :return: prediction and prediction uncertainty :rtype: ndarry :History: 2017-Dec-06 - Written - Henry Leung (University of Toronto) """ self.has_model_check() input_data = self.pre_testing_checklist_master(input_data) input_array = self.input_normalizer.normalize(input_data, calc=False) total_test_num = input_data['input'].shape[0] # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: self.batch_size = total_test_num # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // self.batch_size) * self.batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator # TODO: named output???? predictions = np.zeros((total_test_num, self._labels_shape['output'])) norm_data_main = {} norm_data_remainder = {} for name in input_array.keys(): norm_data_main.update({name: input_array[name][:data_gen_shape]}) norm_data_remainder.update({name: input_array[name][data_gen_shape:]}) norm_data_main = self._tensor_dict_sanitize(norm_data_main, self.keras_model.input_names) norm_data_remainder = self._tensor_dict_sanitize(norm_data_remainder, self.keras_model.input_names) # Data Generator for prediction with tqdm(total=total_test_num, unit="sample") as pbar: prediction_generator = CNNPredDataGenerator(batch_size=self.batch_size, shuffle=False, steps_per_epoch=total_test_num // self.batch_size, data=[norm_data_main], pbar=pbar) predictions[:data_gen_shape] = np.asarray(self.keras_model.predict(prediction_generator)) if remainder_shape != 0: remainder_generator = CNNPredDataGenerator(batch_size=remainder_shape, shuffle=False, steps_per_epoch=1, data=[norm_data_remainder], pbar=pbar) predictions[data_gen_shape:] = np.asarray(self.keras_model.predict(remainder_generator)) if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize(list_to_dict(self.keras_model.output_names, predictions)) else: predictions *= self.labels_std predictions += self.labels_mean return predictions['output']
def mode_checker(self, data): if type(data) is not dict: dict_flag = False data = {"Temp": data} self.mean_labels = {"Temp": self.mean_labels} self.std_labels = {"Temp": self.std_labels} else: dict_flag = True master_data = {} if type(self.normalization_mode) is not dict: self.normalization_mode = list_to_dict( data.keys(), to_iterable(self.normalization_mode)) for name in data.keys(): # normalize data for each named inputs if data[name].ndim == 1: data_array = np.expand_dims(data[name], 1) else: data_array = np.array(data[name]) self.normalization_mode.update({ name: str(self.normalization_mode[name]) }) # just to prevent unnecessary type issue if data_array.dtype == bool: if self.normalization_mode[ name] != '0': # binary classification case warnings.warn( "Data type is detected as bool, setting normalization_mode to 0 which is " "doing nothing because no normalization can be done on bool" ) self.normalization_mode[name] = '0' data_array = data_array.astype( np.float32, copy=False) # need to convert data to float in every case if self.normalization_mode[name] == '0': self.featurewise_center.update({name: False}) self.datasetwise_center.update({name: False}) self.featurewise_stdalization.update({name: False}) self.datasetwise_stdalization.update({name: False}) self.mean_labels.update({name: np.array([0.])}) self.std_labels.update({name: np.array([1.])}) elif self.normalization_mode[name] == '1': self.featurewise_center.update({name: False}) self.datasetwise_center.update({name: True}) self.featurewise_stdalization.update({name: False}) self.datasetwise_stdalization.update({name: True}) elif self.normalization_mode[name] == '2': self.featurewise_center.update({name: True}) self.datasetwise_center.update({name: False}) self.featurewise_stdalization.update({name: True}) self.datasetwise_stdalization.update({name: False}) elif self.normalization_mode[name] == '3': self.featurewise_center.update({name: True}) self.datasetwise_center.update({name: False}) self.featurewise_stdalization.update({name: False}) self.datasetwise_stdalization.update({name: False}) elif self.normalization_mode[ name] == '3s': # allow custom function, default to use sigmoid to normalize self.featurewise_center.update({name: False}) self.datasetwise_center.update({name: False}) self.featurewise_stdalization.update({name: False}) self.datasetwise_stdalization.update({name: False}) if self._custom_norm_func is None: self._custom_norm_func = sigmoid if self._custom_denorm_func is None: self._custom_denorm_func = sigmoid_inv self.mean_labels.update({name: np.array([0.])}) self.std_labels.update({name: np.array([1.])}) elif self.normalization_mode[name] == '4': self.featurewise_center.update({name: False}) self.datasetwise_center.update({name: False}) self.featurewise_stdalization.update({name: True}) self.datasetwise_stdalization.update({name: False}) elif self.normalization_mode[name] == '255': # Used to normalize 8bit images self.featurewise_center.update({name: False}) self.datasetwise_center.update({name: False}) self.featurewise_stdalization.update({name: False}) self.datasetwise_stdalization.update({name: False}) self.mean_labels.update({name: np.array([0.])}) self.std_labels.update({name: np.array([255.])}) else: raise ValueError( f"Unknown Mode -> {self.normalization_mode[name]}") master_data.update({name: data_array}) return master_data, dict_flag
def load_folder(folder=None): """ To load astroNN model object from folder :param folder: [optional] you should provide folder name if outside folder, do not specific when you are inside the folder :type folder: str :return: astroNN Neural Network instance :rtype: astroNN.nn.NeuralNetMaster.NeuralNetMaster :History: 2017-Dec-29 - Written - Henry Leung (University of Toronto) """ currentdir = os.getcwd() if folder is not None: fullfilepath = os.path.join(currentdir, folder) else: fullfilepath = currentdir astronn_model_obj = None if folder is not None and os.path.isfile( os.path.join(folder, 'astroNN_model_parameter.json')) is True: with open(os.path.join(folder, 'astroNN_model_parameter.json')) as f: parameter = json.load(f) f.close() elif os.path.isfile('astroNN_model_parameter.json') is True: with open('astroNN_model_parameter.json') as f: parameter = json.load(f) f.close() elif folder is not None and not os.path.exists(folder): raise IOError('Folder not exists: ' + str(currentdir + os.sep + folder)) else: raise FileNotFoundError( 'Are you sure this is an astroNN generated folder?') identifier = parameter['id'] unknown_model_message = f'Unknown model identifier -> {identifier}!' # need to point to the actual neural network if non-travial location if identifier == 'Galaxy10CNN': astronn_model_obj = Galaxy10CNN() else: # else try to import it from standard way try: astronn_model_obj = getattr( importlib.import_module(f"astroNN.models"), identifier)() except ImportError: # try to load custom model from CUSTOM_MODEL_PATH if none are working CUSTOM_MODEL_PATH = custom_model_path_reader() # try the current folder and see if there is any .py on top of CUSTOM_MODEL_PATH list_py_files = [ os.path.join(fullfilepath, f) for f in os.listdir(fullfilepath) if f.endswith(".py") ] if CUSTOM_MODEL_PATH is None and list_py_files is None: print("\n") raise TypeError(unknown_model_message) else: for path_list in ( path_list for path_list in [CUSTOM_MODEL_PATH, list_py_files] if path_list is not None): for path in path_list: head, tail = os.path.split(path) sys.path.insert(0, head) try: model = getattr( importlib.import_module(tail.strip('.py')), str(identifier)) astronn_model_obj = model() except AttributeError: pass if astronn_model_obj is None: print("\n") raise TypeError(unknown_model_message) astronn_model_obj.currentdir = currentdir astronn_model_obj.fullfilepath = fullfilepath astronn_model_obj.folder_name = folder if folder is not None else os.path.basename( os.path.normpath(currentdir)) # Must have parameter astronn_model_obj._input_shape = parameter['input'] astronn_model_obj._labels_shape = parameter['labels'] if type(astronn_model_obj._input_shape) is not dict: astronn_model_obj._input_shape = { "input": astronn_model_obj._input_shape } if type(astronn_model_obj._labels_shape) is not dict: astronn_model_obj._labels_shape = { "output": astronn_model_obj._labels_shape } astronn_model_obj.num_hidden = parameter['hidden'] astronn_model_obj.input_norm_mode = parameter['input_norm_mode'] astronn_model_obj.labels_norm_mode = parameter['labels_norm_mode'] astronn_model_obj.batch_size = parameter['batch_size'] astronn_model_obj.targetname = parameter['targetname'] astronn_model_obj.val_size = parameter['valsize'] # Conditional parameter depends on neural net architecture try: astronn_model_obj.num_filters = parameter['filternum'] except KeyError: pass try: astronn_model_obj.filter_len = parameter['filterlen'] except KeyError: pass try: pool_length = parameter['pool_length'] if pool_length is not None: if isinstance(pool_length, int): # multi-dimensional case astronn_model_obj.pool_length = parameter['pool_length'] else: astronn_model_obj.pool_length = list(parameter['pool_length']) except KeyError or TypeError: pass try: # need to convert to int because of keras do not want array or list astronn_model_obj.latent_dim = int(parameter['latent']) except KeyError: pass try: astronn_model_obj.task = parameter['task'] except KeyError: pass try: astronn_model_obj.dropout_rate = parameter['dropout_rate'] except KeyError: pass try: # if inverse model precision exists, so does length_scale astronn_model_obj.inv_model_precision = parameter['inv_tau'] astronn_model_obj.length_scale = parameter['length_scale'] except KeyError: pass try: astronn_model_obj.l1 = parameter['l1'] except KeyError: pass try: astronn_model_obj.l2 = parameter['l2'] except KeyError: pass try: astronn_model_obj.maxnorm = parameter['maxnorm'] except KeyError: pass try: astronn_model_obj.input_names = parameter['input_names'] except KeyError: astronn_model_obj.input_names = ['input'] try: astronn_model_obj.output_names = parameter['output_names'] except KeyError: astronn_model_obj.output_names = ['output'] try: astronn_model_obj._last_layer_activation = parameter[ 'last_layer_activation'] except KeyError: pass try: astronn_model_obj.activation = parameter['activation'] except KeyError: pass with h5py.File(os.path.join(astronn_model_obj.fullfilepath, 'model_weights.h5'), mode='r') as f: training_config = json.loads(f.attrs['training_config']) optimizer_config = training_config['optimizer_config'] optimizer = optimizers.deserialize(optimizer_config) model_config = json.loads(f.attrs['model_config']) # input/name names, mean, std input_names = [] output_names = [] for lay in model_config["config"]["input_layers"]: input_names.append(lay[0]) for lay in model_config["config"]["output_layers"]: output_names.append(lay[0]) astronn_model_obj.input_mean = list_to_dict( input_names, dict_list_to_dict_np(parameter['input_mean'])) astronn_model_obj.labels_mean = list_to_dict( output_names, dict_list_to_dict_np(parameter['labels_mean'])) astronn_model_obj.input_std = list_to_dict( input_names, dict_list_to_dict_np(parameter['input_std'])) astronn_model_obj.labels_std = list_to_dict( output_names, dict_list_to_dict_np(parameter['labels_std'])) # Recover loss functions and metrics. losses_raw = convert_custom_objects(training_config['loss']) try: try: loss = [ losses_lookup(losses_raw[_loss]) for _loss in losses_raw ] except TypeError: loss = losses_lookup(losses_raw) except: raise LookupError("Cant lookup loss") metrics_raw = convert_custom_objects(training_config['metrics']) # its weird that keras needs -> metrics[metric][0] instead of metrics[metric] likes losses try: try: if version.parse(tf.__version__) >= version.parse("2.4.0"): metrics = [ losses_lookup(_metric['config']['fn']) for _metric in metrics_raw[0] ] else: metrics = [ losses_lookup(metrics_raw[_metric][0]) for _metric in metrics_raw ] except TypeError: metrics = [losses_lookup(metrics_raw[0])] except: metrics = metrics_raw sample_weight_mode = training_config['sample_weight_mode'] if \ hasattr(training_config, 'sample_weight_mode') else None loss_weights = training_config['loss_weights'] weighted_metrics = None # compile the model astronn_model_obj.compile(optimizer=optimizer, loss=loss, metrics=metrics, weighted_metrics=weighted_metrics, loss_weights=loss_weights, sample_weight_mode=sample_weight_mode) # set weights astronn_model_obj.keras_model.load_weights( os.path.join(astronn_model_obj.fullfilepath, 'model_weights.h5')) # Build train function (to get weight updates), need to consider Sequential model too astronn_model_obj.keras_model.make_train_function() try: optimizer_weights_group = f['optimizer_weights'] if version.parse(h5py.__version__) >= version.parse("3.0"): optimizer_weight_names = [ n for n in optimizer_weights_group.attrs['weight_names'] ] else: optimizer_weight_names = [ n.decode('utf8') for n in optimizer_weights_group.attrs['weight_names'] ] optimizer_weight_values = [ optimizer_weights_group[n] for n in optimizer_weight_names ] astronn_model_obj.keras_model.optimizer._create_all_weights( astronn_model_obj.keras_model.trainable_variables) astronn_model_obj.keras_model.optimizer.set_weights( optimizer_weight_values) except KeyError: warnings.warn( "Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer." ) # create normalizer and set correct mean and std astronn_model_obj.input_normalizer = Normalizer( mode=astronn_model_obj.input_norm_mode) astronn_model_obj.labels_normalizer = Normalizer( mode=astronn_model_obj.labels_norm_mode) astronn_model_obj.input_normalizer.mean_labels = astronn_model_obj.input_mean astronn_model_obj.input_normalizer.std_labels = astronn_model_obj.input_std astronn_model_obj.labels_normalizer.mean_labels = astronn_model_obj.labels_mean astronn_model_obj.labels_normalizer.std_labels = astronn_model_obj.labels_std print("========================================================") print( f"Loaded astroNN model, model type: {astronn_model_obj.name} -> {identifier}" ) print("========================================================") return astronn_model_obj
def evaluate(self, input_data, labels, inputs_err=None, labels_err=None): """ Evaluate neural network by provided input data and labels and get back a metrics score :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :return: metrics score dictionary :rtype: dict :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) input_data = {"input": input_data} labels = {"output": labels} # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_input_err = inputs_err / self.input_std['input'] norm_labels_err = labels_err / self.labels_std['output'] norm_data.update({ "input_err": norm_input_err, "labels_err": norm_labels_err }) norm_labels.update({ "labels_err": norm_labels_err, "variance_output": norm_labels["output"] }) total_num = input_data['input'].shape[0] eval_batchsize = self.batch_size if total_num > self.batch_size else total_num steps = total_num // self.batch_size if total_num > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") evaluate_generator = BayesianCNNDataGenerator( batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels]) scores = self.keras_model.evaluate_generator(evaluate_generator) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = self.keras_model.metrics_names print( f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed' ) return list_to_dict(funcname, scores)
def test(self, input_data, inputs_err=None): """ Test model, High performance version designed for fast variational inference on GPU :param input_data: Data to be inferred with neural network :type input_data: ndarray :param inputs_err: Error for input_data, same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :return: prediction and prediction uncertainty :History: | 2018-Jan-06 - Written - Henry Leung (University of Toronto) | 2018-Apr-12 - Updated - Henry Leung (University of Toronto) """ self.has_model_check() if gpu_availability() is False and self.mc_num > 25: warnings.warn( f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can ' f'potentially be very slow! \n ' f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n' f'This is just a warning, and will not shown if mc_num < 25 on CPU' ) if self.mc_num < 2: raise AttributeError("mc_num cannot be smaller than 2") # if no error array then just zeros if inputs_err is None: inputs_err = np.zeros_like(input_data) else: inputs_err = np.atleast_2d(inputs_err) inputs_err /= self.input_std['input'] input_data = {"input": input_data, "input_err": inputs_err} input_data = self.pre_testing_checklist_master(input_data) if self.input_normalizer is not None: input_array = self.input_normalizer.normalize(input_data, calc=False) else: # Prevent shallow copy issue input_array = np.array(input_data) input_array -= self.input_mean['input'] input_array /= self.input_std['input'] total_test_num = input_data['input'].shape[0] # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: batch_size = total_test_num else: batch_size = self.batch_size # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // batch_size) * batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator norm_data_main = {} norm_data_remainder = {} for name in input_array.keys(): norm_data_main.update({name: input_array[name][:data_gen_shape]}) norm_data_remainder.update( {name: input_array[name][data_gen_shape:]}) start_time = time.time() print("Starting Dropout Variational Inference") # Data Generator for prediction prediction_generator = BayesianCNNPredDataGenerator( batch_size=batch_size, shuffle=False, steps_per_epoch=data_gen_shape // batch_size, data=[norm_data_main]) new = FastMCInference(self.mc_num)(self.keras_model_predict) result = np.asarray(new.predict_generator(prediction_generator)) if remainder_shape != 0: # deal with remainder remainder_generator = BayesianCNNPredDataGenerator( batch_size=remainder_shape, shuffle=False, steps_per_epoch=1, data=[norm_data_remainder]) remainder_result = np.asarray( new.predict_generator(remainder_generator)) if remainder_shape == 1: remainder_result = np.expand_dims(remainder_result, axis=0) result = np.concatenate((result, remainder_result)) # in case only 1 test data point, in such case we need to add a dimension if result.ndim < 3 and batch_size == 1: result = np.expand_dims(result, axis=0) half_first_dim = result.shape[ 1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong predictions = result[:, :half_first_dim, 0] # mean prediction mc_dropout_uncertainty = result[:, :half_first_dim, 1] * ( self.labels_std['output']**2) # model uncertainty predictions_var = np.exp(result[:, half_first_dim:, 0]) * ( self.labels_std['output']**2) # predictive uncertainty print( f'Completed Dropout Variational Inference with {self.mc_num} forward passes, ' f'{(time.time() - start_time):.{2}f}s elapsed') if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize( list_to_dict([self.keras_model.output_names[0]], predictions)) predictions = predictions['output'] else: predictions *= self.labels_std['output'] predictions += self.labels_mean['output'] if self.task == 'regression': # Predictive variance pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty pred_uncertainty = np.sqrt(pred_var) # Convert back to std error # final correction from variance to standard derivation mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty) predictive_uncertainty = np.sqrt(predictions_var) elif self.task == 'classification': # we want entropy for classification uncertainty predicted_class = np.argmax(predictions, axis=1) mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float) predictive_uncertainty = np.ones_like(predicted_class, dtype=float) # center variance predictions_var -= 1. for i in range(predicted_class.shape[0]): all_prediction = np.array(predictions[i, :]) mc_dropout_uncertainty[i] = -np.sum( all_prediction * np.log(all_prediction)) predictive_uncertainty[i] = predictions_var[i, predicted_class[i]] pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty # We only want the predicted class back predictions = predicted_class elif self.task == 'binary_classification': # we want entropy for classification uncertainty, so need prediction in logits space mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions), axis=0) # need to activate before round to int so that the prediction is always 0 or 1 predictions = np.rint(sigmoid(predictions)) predictive_uncertainty = predictions_var pred_uncertainty = mc_dropout_uncertainty + predictions_var else: raise AttributeError('Unknown Task') return predictions, { 'total': pred_uncertainty, 'model': mc_dropout_uncertainty, 'predictive': predictive_uncertainty }
def predict(self, input_data): """ Use the neural network to do inference and get reconstructed data :param input_data: Data to be inferred with neural network :type input_data: ndarray :return: reconstructed data :rtype: ndarry :History: 2017-Dec-06 - Written - Henry Leung (University of Toronto) """ input_data = self.pre_testing_checklist_master(input_data) if self.input_normalizer is not None: input_array = self.input_normalizer.normalize(input_data, calc=False) else: # Prevent shallow copy issue input_array = np.array(input_data) input_array -= self.input_mean input_array /= self.input_std total_test_num = input_data['input'].shape[0] # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: self.batch_size = total_test_num # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // self.batch_size) * self.batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator predictions = np.zeros((total_test_num, self._labels_shape['output'], 1)) norm_data_main = {} norm_data_remainder = {} for name in input_array.keys(): norm_data_main.update({name: input_array[name][:data_gen_shape]}) norm_data_remainder.update({name: input_array[name][data_gen_shape:]}) norm_data_main = self._tensor_dict_sanitize(norm_data_main, self.keras_model.input_names) norm_data_remainder = self._tensor_dict_sanitize(norm_data_remainder, self.keras_model.input_names) start_time = time.time() print("Starting Inference") # Data Generator for prediction with tqdm(total=total_test_num, unit="sample") as pbar: prediction_generator = CVAEPredDataGenerator(batch_size=self.batch_size, shuffle=False, steps_per_epoch=total_test_num // self.batch_size, data=[norm_data_main]) result = np.asarray(self.keras_model.predict(prediction_generator)) if remainder_shape != 0: remainder_generator = CVAEPredDataGenerator(batch_size=remainder_shape, shuffle=False, steps_per_epoch=1, data=[norm_data_remainder], pbar=pbar) remainder_result = np.asarray(self.keras_model.predict(remainder_generator)) result = np.concatenate((result, remainder_result)) if self.labels_normalizer is not None: # TODO: handle named output in the future predictions[:, :, 0] = self.labels_normalizer.denormalize(list_to_dict(self.keras_model.output_names, predictions[:, :, 0]))['output'] else: predictions[:, :, 0] *= self.labels_std predictions[:, :, 0] += self.labels_mean print(f'Completed Inference, {(time.time() - start_time):.{2}f}s elapsed') return predictions
def jacobian_old(self, x=None, mean_output=False, denormalize=False): """ | Calculate jacobian of gradient of output to input | | Please notice that the de-normalize (if True) assumes the output depends on the input data first orderly | in which the equation is simply jacobian divided the input scaling :param x: Input Data :type x: ndarray :param mean_output: False to get all jacobian, True to get the mean :type mean_output: boolean :param denormalize: De-normalize Jacobian :type denormalize: bool :History: 2017-Nov-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if x is None: raise ValueError('Please provide data to calculate the jacobian') x = list_to_dict(self.keras_model.input_names, x) if self.input_normalizer is not None: x_data = self.input_normalizer.normalize(x, calc=False) x_data = x_data['input'] else: # Prevent shallow copy issue x_data = np.array(x) x_data -= self.input_mean x_data /= self.input_std try: input_tens = self.keras_model_predict.get_layer("input").input output_tens = self.keras_model_predict.get_layer("output").output input_shape_expectation = self.keras_model_predict.get_layer( "input").input_shape except AttributeError: input_tens = self.keras_model.get_layer("input").input output_tens = self.keras_model.get_layer("output").output input_shape_expectation = self.keras_model.get_layer( "input").input_shape start_time = time.time() if len(input_shape_expectation) == 1: input_shape_expectation = input_shape_expectation[0] if len(input_shape_expectation) == 3: x_data = np.atleast_3d(x_data) grad_list = [] for j in range(self._labels_shape['output']): grad_list.append(tf.gradients(output_tens[0, j], input_tens)) final_stack = tf.stack(tf.squeeze(grad_list)) jacobian = np.ones((x_data.shape[0], self._labels_shape['output'], x_data.shape[1]), dtype=np.float32) for i in range(x_data.shape[0]): x_in = x_data[i:i + 1] jacobian[i, :, :] = get_session().run( final_stack, feed_dict={ input_tens: x_in, tfk.backend.learning_phase(): 0 }) elif len(input_shape_expectation) == 4: monoflag = False if len(x_data.shape) < 4: monoflag = True x_data = x_data[:, :, :, np.newaxis] jacobian = np.ones( (x_data.shape[0], self._labels_shape['output'], x_data.shape[1], x_data.shape[2], x_data.shape[3]), dtype=np.float32) grad_list = [] for j in range(self._labels_shape['output']): grad_list.append( tf.gradients( self.keras_model.get_layer("output").output[0, j], input_tens)) final_stack = tf.stack(tf.squeeze(grad_list)) for i in range(x_data.shape[0]): x_in = x_data[i:i + 1] if monoflag is False: jacobian[i, :, :, :, :] = get_session().run( final_stack, feed_dict={ input_tens: x_in, tfk.backend.learning_phase(): 0 }) else: jacobian[i, :, :, :, 0] = get_session().run( final_stack, feed_dict={ input_tens: x_in, tfk.backend.learning_phase(): 0 }) else: raise ValueError( 'Input data shape do not match neural network expectation') if mean_output is True: jacobian_master = np.mean(jacobian, axis=0) else: jacobian_master = np.array(jacobian) jacobian_master = np.squeeze(jacobian_master) if denormalize: if self.input_std is not None: jacobian_master = jacobian_master / self.input_std if self.labels_std is not None: try: jacobian_master = jacobian_master * self.labels_std except ValueError: jacobian_master = jacobian_master * self.labels_std.reshape( -1, 1) print( f'Finished gradient calculation, {(time.time() - start_time):.{2}f} seconds elapsed' ) return jacobian_master
def predict_dataset(self, file): class BayesianCNNPredDataGeneratorV2(GeneratorMaster): def __init__(self, batch_size, shuffle, steps_per_epoch, manual_reset=False, pbar=None, nn_model=None): super().__init__(batch_size=batch_size, shuffle=shuffle, steps_per_epoch=steps_per_epoch, data=None, manual_reset=manual_reset) self.pbar = pbar # initial idx self.idx_list = self._get_exploration_order(range(len(file))) self.current_idx = 0 self.nn_model = nn_model def _data_generation(self, idx_list_temp): # Generate data inputs = self.nn_model.input_normalizer.normalize( { "input": file[idx_list_temp], "input_err": np.zeros_like(file[idx_list_temp]) }, calc=False) x = self.input_d_checking(inputs, np.arange(len(idx_list_temp))) return x def __getitem__(self, index): x = self._data_generation( self.idx_list[index * self.batch_size:(index + 1) * self.batch_size]) if self.pbar: self.pbar.update(self.batch_size) return x def on_epoch_end(self): # shuffle the list when epoch ends for the next epoch self.idx_list = self._get_exploration_order(range(len(file))) self.has_model_check() if gpu_availability() is False and self.mc_num > 25: warnings.warn( f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can ' f'potentially be very slow! \n ' f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n' f'This is just a warning, and will not shown if mc_num < 25 on CPU' ) if self.mc_num < 2: raise AttributeError("mc_num cannot be smaller than 2") total_test_num = len(file) # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: batch_size = total_test_num else: batch_size = self.batch_size # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // batch_size) * batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator # Data Generator for prediction with tqdm(total=total_test_num, unit="sample") as pbar: pbar.set_postfix({'Monte-Carlo': self.mc_num}) # suppress pfor warning from TF old_level = tf.get_logger().level tf.get_logger().setLevel('ERROR') prediction_generator = BayesianCNNPredDataGeneratorV2( batch_size=batch_size, shuffle=False, steps_per_epoch=data_gen_shape // batch_size, pbar=pbar, nn_model=self) new = FastMCInference(self.mc_num)(self.keras_model_predict) result = np.asarray(new.predict(prediction_generator)) if remainder_shape != 0: # deal with remainder remainder_generator = BayesianCNNPredDataGeneratorV2( batch_size=remainder_shape, shuffle=False, steps_per_epoch=1, pbar=pbar, nn_model=self) remainder_result = np.asarray(new.predict(remainder_generator)) if remainder_shape == 1: remainder_result = np.expand_dims(remainder_result, axis=0) result = np.concatenate((result, remainder_result)) tf.get_logger().setLevel(old_level) # in case only 1 test data point, in such case we need to add a dimension if result.ndim < 3 and batch_size == 1: result = np.expand_dims(result, axis=0) half_first_dim = result.shape[ 1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong predictions = result[:, :half_first_dim, 0] # mean prediction mc_dropout_uncertainty = result[:, :half_first_dim, 1] * ( self.labels_std['output']**2) # model uncertainty predictions_var = np.exp(result[:, half_first_dim:, 0]) * ( self.labels_std['output']**2) # predictive uncertainty if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize( list_to_dict([self.keras_model.output_names[0]], predictions)) predictions = predictions['output'] else: predictions *= self.labels_std['output'] predictions += self.labels_mean['output'] if self.task == 'regression': # Predictive variance pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty pred_uncertainty = np.sqrt(pred_var) # Convert back to std error # final correction from variance to standard derivation mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty) predictive_uncertainty = np.sqrt(predictions_var) elif self.task == 'classification': # we want entropy for classification uncertainty predicted_class = np.argmax(predictions, axis=1) mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float) predictive_uncertainty = np.ones_like(predicted_class, dtype=float) # center variance predictions_var -= 1. for i in range(predicted_class.shape[0]): all_prediction = np.array(predictions[i, :]) mc_dropout_uncertainty[i] = -np.sum( all_prediction * np.log(all_prediction)) predictive_uncertainty[i] = predictions_var[i, predicted_class[i]] pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty # We only want the predicted class back predictions = predicted_class elif self.task == 'binary_classification': # we want entropy for classification uncertainty, so need prediction in logits space mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions), axis=0) # need to activate before round to int so that the prediction is always 0 or 1 predictions = np.rint(sigmoid(predictions)) predictive_uncertainty = predictions_var pred_uncertainty = mc_dropout_uncertainty + predictions_var else: raise AttributeError('Unknown Task') return predictions, { 'total': pred_uncertainty, 'model': mc_dropout_uncertainty, 'predictive': predictive_uncertainty }