def load_folder(folder=None): """ To load astroNN model object from folder :param folder: [optional] you should provide folder name if outside folder, do not specific when you are inside the folder :type folder: str :return: astroNN Neural Network instance :rtype: astroNN.nn.NeuralNetMaster.NeuralNetMaster :History: 2017-Dec-29 - Written - Henry Leung (University of Toronto) """ currentdir = os.getcwd() if folder is not None: fullfilepath = os.path.join(currentdir, folder) else: fullfilepath = currentdir astronn_model_obj = None if folder is not None and os.path.isfile( os.path.join(folder, 'astroNN_model_parameter.json')) is True: with open(os.path.join(folder, 'astroNN_model_parameter.json')) as f: parameter = json.load(f) f.close() elif os.path.isfile('astroNN_model_parameter.json') is True: with open('astroNN_model_parameter.json') as f: parameter = json.load(f) f.close() elif folder is not None and not os.path.exists(folder): raise IOError('Folder not exists: ' + str(currentdir + os.sep + folder)) else: raise FileNotFoundError( 'Are you sure this is an astroNN generated folder?') identifier = parameter['id'] unknown_model_message = f'Unknown model identifier -> {identifier}!' # need to point to the actual neural network if non-travial location if identifier == 'Galaxy10CNN': astronn_model_obj = Galaxy10CNN() else: # else try to import it from standard way try: astronn_model_obj = getattr( importlib.import_module(f"astroNN.models"), identifier)() except ImportError: # try to load custom model from CUSTOM_MODEL_PATH if none are working CUSTOM_MODEL_PATH = custom_model_path_reader() # try the current folder and see if there is any .py on top of CUSTOM_MODEL_PATH list_py_files = [ os.path.join(fullfilepath, f) for f in os.listdir(fullfilepath) if f.endswith(".py") ] if CUSTOM_MODEL_PATH is None and list_py_files is None: print("\n") raise TypeError(unknown_model_message) else: for path_list in ( path_list for path_list in [CUSTOM_MODEL_PATH, list_py_files] if path_list is not None): for path in path_list: head, tail = os.path.split(path) sys.path.insert(0, head) try: model = getattr( importlib.import_module(tail.strip('.py')), str(identifier)) astronn_model_obj = model() except AttributeError: pass if astronn_model_obj is None: print("\n") raise TypeError(unknown_model_message) astronn_model_obj.currentdir = currentdir astronn_model_obj.fullfilepath = fullfilepath astronn_model_obj.folder_name = folder if folder is not None else os.path.basename( os.path.normpath(currentdir)) # Must have parameter astronn_model_obj._input_shape = parameter['input'] astronn_model_obj._labels_shape = parameter['labels'] astronn_model_obj.num_hidden = parameter['hidden'] astronn_model_obj.input_norm_mode = parameter['input_norm_mode'] astronn_model_obj.labels_norm_mode = parameter['labels_norm_mode'] astronn_model_obj.input_mean = np.array(parameter['input_mean']) astronn_model_obj.labels_mean = np.array(parameter['labels_mean']) astronn_model_obj.input_std = np.array(parameter['input_std']) astronn_model_obj.labels_std = np.array(parameter['labels_std']) astronn_model_obj.batch_size = parameter['batch_size'] astronn_model_obj.targetname = parameter['targetname'] astronn_model_obj.val_size = parameter['valsize'] # create normalizer and set correct mean and std astronn_model_obj.input_normalizer = Normalizer( mode=astronn_model_obj.input_norm_mode) astronn_model_obj.labels_normalizer = Normalizer( mode=astronn_model_obj.labels_norm_mode) astronn_model_obj.input_normalizer.mean_labels = astronn_model_obj.input_mean astronn_model_obj.input_normalizer.std_labels = astronn_model_obj.input_std astronn_model_obj.labels_normalizer.mean_labels = astronn_model_obj.labels_mean astronn_model_obj.labels_normalizer.std_labels = astronn_model_obj.labels_std # Conditional parameter depends on neural net architecture try: astronn_model_obj.num_filters = parameter['filternum'] except KeyError: pass try: astronn_model_obj.filter_len = parameter['filterlen'] except KeyError: pass try: pool_length = parameter['pool_length'] if pool_length is not None: if isinstance(pool_length, int): # multi-dimensional case astronn_model_obj.pool_length = parameter['pool_length'] else: astronn_model_obj.pool_length = list(parameter['pool_length']) except KeyError or TypeError: pass try: # need to convert to int because of keras do not want array or list astronn_model_obj.latent_dim = int(parameter['latent']) except KeyError: pass try: astronn_model_obj.task = parameter['task'] except KeyError: pass try: astronn_model_obj.dropout_rate = parameter['dropout_rate'] except KeyError: pass try: # if inverse model precision exists, so does length_scale astronn_model_obj.inv_model_precision = parameter['inv_tau'] astronn_model_obj.length_scale = parameter['length_scale'] except KeyError: pass try: astronn_model_obj.l1 = parameter['l1'] except KeyError: pass try: astronn_model_obj.l2 = parameter['l2'] except KeyError: pass try: astronn_model_obj.maxnorm = parameter['maxnorm'] except KeyError: pass try: astronn_model_obj._last_layer_activation = parameter[ 'last_layer_activation'] except KeyError: pass try: astronn_model_obj.activation = parameter['activation'] except KeyError: pass global _GRAPH_COUTNER global _GRAPH_STORAGE global _SESSION_STORAGE _GRAPH_COUTNER += 1 _GRAPH_STORAGE.append(get_default_graph()) # only 2 cases as thats all I can think of will happen if get_default_session() is not None: session = get_default_session() elif keras.backend.get_session() is not None: session = keras.backend.get_session() else: session = None _SESSION_STORAGE.append(session) with h5py.File(os.path.join(astronn_model_obj.fullfilepath, 'model_weights.h5'), mode='r') as f: training_config = f.attrs.get('training_config') training_config = json.loads(training_config.decode('utf-8')) optimizer_config = training_config['optimizer_config'] optimizer = optimizers.deserialize(optimizer_config) # Recover loss functions and metrics. losses_raw = convert_custom_objects(training_config['loss']) try: try: loss = [ losses_lookup(losses_raw[_loss]) for _loss in losses_raw ] except TypeError: loss = losses_lookup(losses_raw) except: pass metrics_raw = convert_custom_objects(training_config['metrics']) # its weird that keras needs -> metrics[metric][0] instead of metrics[metric] likes losses try: try: metrics = [ losses_lookup(metrics_raw[_metric][0]) for _metric in metrics_raw ] except TypeError: metrics = [losses_lookup(metrics_raw[0])] except: pass sample_weight_mode = training_config['sample_weight_mode'] loss_weights = training_config['loss_weights'] weighted_metrics = None # compile the model astronn_model_obj.compile(optimizer=optimizer, loss=loss, metrics=metrics, weighted_metrics=weighted_metrics, loss_weights=loss_weights, sample_weight_mode=sample_weight_mode) # set weights astronn_model_obj.keras_model.load_weights( os.path.join(astronn_model_obj.fullfilepath, 'model_weights.h5')) # Build train function (to get weight updates), need to consider Sequential model too astronn_model_obj.keras_model._make_train_function() optimizer_weights_group = f['optimizer_weights'] optimizer_weight_names = [ n.decode('utf8') for n in optimizer_weights_group.attrs['weight_names'] ] optimizer_weight_values = [ optimizer_weights_group[n] for n in optimizer_weight_names ] astronn_model_obj.keras_model.optimizer.set_weights( optimizer_weight_values) astronn_model_obj.graph = _GRAPH_STORAGE[ _GRAPH_COUTNER - 1] # the graph associated with the model astronn_model_obj.session = _SESSION_STORAGE[ _GRAPH_COUTNER - 1] # the model associated with the model print("========================================================") print( f"Loaded astroNN model, model type: {astronn_model_obj.name} -> {identifier}" ) print("========================================================") return astronn_model_obj
def train_on_batch(self, input_data, labels, inputs_err=None, labels_err=None): """ Train a Bayesian neural network by running a single gradient update on all of your data, suitable for fine-tuning :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :return: None :rtype: NoneType :History: | 2018-Aug-25 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_input_err = inputs_err / self.input_std norm_labels_err = labels_err / self.labels_std start_time = time.time() fit_generator = BayesianCNNDataGenerator( batch_size=input_data.shape[0], shuffle=False, steps_per_epoch=1, data=[norm_data, norm_labels, norm_input_err, norm_labels_err]) score = self.keras_model.fit_generator( fit_generator, epochs=1, verbose=self.verbose, workers=os.cpu_count(), use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training on Batch, {(time.time() - start_time):.{2}f}s in total' ) return None
def evaluate(self, input_data, labels, inputs_err=None, labels_err=None): """ Evaluate neural network by provided input data and labels and get back a metrics score :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :return: metrics score dictionary :rtype: dict :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_input_err = inputs_err / self.input_std norm_labels_err = labels_err / self.labels_std eval_batchsize = self.batch_size if input_data.shape[ 0] > self.batch_size else input_data.shape[0] steps = input_data.shape[0] // self.batch_size if input_data.shape[ 0] > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") evaluate_generator = BayesianCNNDataGenerator( batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels, norm_input_err, norm_labels_err]) scores = self.keras_model.evaluate_generator(evaluate_generator) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = [] if isinstance(self.keras_model.metrics, dict): func_list = self.keras_model.metrics[outputname[0]] else: func_list = self.keras_model.metrics for func in func_list: if hasattr(func, __name__): funcname.append(func.__name__) else: funcname.append(func.__class__.__name__) # funcname = [func.__name__ for func in self.keras_model.metrics[outputname[0]]] loss_outputname = ['loss_' + name for name in outputname] output_funcname = [outputname[0] + '_' + name for name in funcname] list_names = ['loss', *loss_outputname, *output_funcname] print( f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed' ) return {name: score for name, score in zip(list_names, scores)}
class CNNBase(NeuralNetMaster, ABC): """Top-level class for a convolutional neural network""" def __init__(self): """ NAME: __init__ PURPOSE: To define astroNN convolutional neural network HISTORY: 2018-Jan-06 - Written - Henry Leung (University of Toronto) """ super().__init__() self.name = 'Convolutional Neural Network' self._model_type = 'CNN' self._model_identifier = None self.initializer = None self.activation = None self._last_layer_activation = None self.num_filters = None self.filter_len = None self.pool_length = None self.num_hidden = None self.reduce_lr_epsilon = None self.reduce_lr_min = None self.reduce_lr_patience = None self.l1 = None self.l2 = None self.maxnorm = None self.dropout_rate = 0.0 self.val_size = 0.1 self.early_stopping_min_delta = 0.0001 self.early_stopping_patience = 4 self.input_norm_mode = 1 self.labels_norm_mode = 2 def compile(self, optimizer=None, loss=None, metrics=None, weighted_metrics=None, loss_weights=None, sample_weight_mode=None): if optimizer is not None: self.optimizer = optimizer elif self.optimizer is None or self.optimizer == 'adam': self.optimizer = Adam(lr=self.lr, beta_1=self.beta_1, beta_2=self.beta_2, epsilon=self.optimizer_epsilon, decay=0.0) if self.task == 'regression': self._last_layer_activation = 'linear' loss_func = mean_squared_error if not loss else loss self.metrics = [mean_absolute_error, mean_error ] if not (metrics, self.metrics) else metrics elif self.task == 'classification': self._last_layer_activation = 'softmax' loss_func = categorical_crossentropy if not loss else loss self.metrics = [categorical_accuracy ] if not (metrics, self.metrics) else metrics elif self.task == 'binary_classification': self._last_layer_activation = 'sigmoid' loss_func = binary_crossentropy if not loss else loss self.metrics = [binary_accuracy( from_logits=False)] if not (metrics, self.metrics) else metrics else: raise RuntimeError( 'Only "regression", "classification" and "binary_classification" are supported' ) self.keras_model = self.model() self.keras_model.compile(loss=loss_func, optimizer=self.optimizer, metrics=self.metrics, weighted_metrics=weighted_metrics, loss_weights=loss_weights, sample_weight_mode=sample_weight_mode) return None def pre_training_checklist_child(self, input_data, labels): self.pre_training_checklist_master(input_data, labels) # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) if self.keras_model is None: # only compiler if there is no keras_model, e.g. fine-tuning does not required self.compile() self.train_idx, self.val_idx = train_test_split( np.arange(self.num_train + self.val_num), test_size=self.val_size) self.training_generator = CNNDataGenerator( batch_size=self.batch_size, shuffle=True, steps_per_epoch=self.num_train // self.batch_size, data=[norm_data[self.train_idx], norm_labels[self.train_idx]], manual_reset=False) val_batchsize = self.batch_size if len( self.val_idx) > self.batch_size else len(self.val_idx) self.validation_generator = CNNDataGenerator( batch_size=val_batchsize, shuffle=False, steps_per_epoch=max(self.val_num // self.batch_size, 1), data=[norm_data[self.val_idx], norm_labels[self.val_idx]], manual_reset=True) return input_data, labels def train(self, input_data, labels): """ Train a Convolutional neural network :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :return: None :rtype: NoneType :History: 2017-Dec-06 - Written - Henry Leung (University of Toronto) """ # Call the checklist to create astroNN folder and save parameters self.pre_training_checklist_child(input_data, labels) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, min_delta=self.reduce_lr_epsilon, patience=self.reduce_lr_patience, min_lr=self.reduce_lr_min, mode='min', verbose=2) early_stopping = EarlyStopping(monitor='val_loss', min_delta=self.early_stopping_min_delta, patience=self.early_stopping_patience, verbose=2, mode='min') self.virtual_cvslogger = VirutalCSVLogger() self.__callbacks = [reduce_lr, self.virtual_cvslogger ] # default must have unchangeable callbacks if self.callbacks is not None: if isinstance(self.callbacks, list): self.__callbacks.extend(self.callbacks) else: self.__callbacks.append(self.callbacks) start_time = time.time() self.history = self.keras_model.fit_generator( generator=self.training_generator, validation_data=self.validation_generator, epochs=self.max_epochs, verbose=self.verbose, workers=os.cpu_count(), callbacks=self.__callbacks, use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training, {(time.time() - start_time):.{2}f}s in total' ) if self.autosave is True: # Call the post training checklist to save parameters self.save() return None def train_on_batch(self, input_data, labels): """ Train a neural network by running a single gradient update on all of your data, suitable for fine-tuning :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :return: None :rtype: NoneType :History: 2018-Aug-22 - Written - Henry Leung (University of Toronto) """ self.has_model_check() # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) start_time = time.time() fit_generator = CNNDataGenerator(batch_size=input_data.shape[0], shuffle=False, steps_per_epoch=1, data=[norm_data, norm_labels]) scores = self.keras_model.fit_generator( generator=fit_generator, epochs=1, verbose=self.verbose, workers=os.cpu_count(), use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training on Batch, {(time.time() - start_time):.{2}f}s in total' ) return None def post_training_checklist_child(self): self.keras_model.save(self.fullfilepath + _astroNN_MODEL_NAME) print(_astroNN_MODEL_NAME + f' saved to {(self.fullfilepath + _astroNN_MODEL_NAME)}') self.hyper_txt.write(f"Dropout Rate: {self.dropout_rate} \n") self.hyper_txt.flush() self.hyper_txt.close() data = { 'id': self.__class__.__name__ if self._model_identifier is None else self._model_identifier, 'pool_length': self.pool_length, 'filterlen': self.filter_len, 'filternum': self.num_filters, 'hidden': self.num_hidden, 'input': self._input_shape, 'labels': self._labels_shape, 'task': self.task, 'last_layer_activation': self._last_layer_activation, 'activation': self.activation, 'input_mean': self.input_mean.tolist(), 'labels_mean': self.labels_mean.tolist(), 'input_std': self.input_std.tolist(), 'labels_std': self.labels_std.tolist(), 'valsize': self.val_size, 'targetname': self.targetname, 'dropout_rate': self.dropout_rate, 'l1': self.l1, 'l2': self.l2, 'maxnorm': self.maxnorm, 'input_norm_mode': self.input_norm_mode, 'labels_norm_mode': self.labels_norm_mode, 'batch_size': self.batch_size } with open(self.fullfilepath + '/astroNN_model_parameter.json', 'w') as f: json.dump(data, f, indent=4, sort_keys=True) def test(self, input_data): """ Use the neural network to do inference :param input_data: Data to be inferred with neural network :type input_data: ndarray :return: prediction and prediction uncertainty :rtype: ndarry :History: 2017-Dec-06 - Written - Henry Leung (University of Toronto) """ self.has_model_check() self.pre_testing_checklist_master() input_data = np.atleast_2d(input_data) if self.input_normalizer is not None: input_array = self.input_normalizer.normalize(input_data, calc=False) else: # Prevent shallow copy issue input_array = np.array(input_data) input_array -= self.input_mean input_array /= self.input_std total_test_num = input_data.shape[0] # Number of testing data # for number of training data smaller than batch_size if input_data.shape[0] < self.batch_size: self.batch_size = input_data.shape[0] # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // self.batch_size) * self.batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator predictions = np.zeros((total_test_num, self._labels_shape)) start_time = time.time() print("Starting Inference") # Data Generator for prediction prediction_generator = CNNPredDataGenerator( batch_size=self.batch_size, shuffle=False, steps_per_epoch=input_array.shape[0] // self.batch_size, data=[input_array[:data_gen_shape]]) predictions[:data_gen_shape] = np.asarray( self.keras_model.predict_generator(prediction_generator)) if remainder_shape != 0: remainder_data = input_array[data_gen_shape:] # assume its caused by mono images, so need to expand dim by 1 if len(input_array[0].shape) != len(self._input_shape): remainder_data = np.expand_dims(remainder_data, axis=-1) result = self.keras_model.predict(remainder_data) predictions[data_gen_shape:] = result.reshape( (remainder_shape, self._labels_shape)) if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize(predictions) else: predictions *= self.labels_std predictions += self.labels_mean print( f'Completed Inference, {(time.time() - start_time):.{2}f}s elapsed' ) return predictions def evaluate(self, input_data, labels): """ Evaluate neural network by provided input data and labels and get back a metrics score :param input_data: Data to be inferred with neural network :type input_data: ndarray :param labels: labels :type labels: ndarray :return: metrics score dictionary :rtype: dict :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) eval_batchsize = self.batch_size if input_data.shape[ 0] > self.batch_size else input_data.shape[0] steps = input_data.shape[0] // self.batch_size if input_data.shape[ 0] > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") evaluate_generator = CNNDataGenerator(batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels]) scores = self.keras_model.evaluate_generator(evaluate_generator) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = [] if isinstance(self.keras_model.metrics, dict): func_list = self.keras_model.metrics[outputname[0]] else: func_list = self.keras_model.metrics for func in func_list: if hasattr(func, __name__): funcname.append(func.__name__) else: funcname.append(func.__class__.__name__) # funcname = [func.__name__ for func in self.keras_model.metrics] output_funcname = [outputname[0] + '_' + name for name in funcname] list_names = ['loss', *output_funcname] print( f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed' ) return {name: score for name, score in zip(list_names, scores)}
class BayesianCNNBase(NeuralNetMaster, ABC): """ Top-level class for a Bayesian convolutional neural network :History: 2018-Jan-06 - Written - Henry Leung (University of Toronto) """ def __init__(self): super().__init__() self.name = 'Bayesian Convolutional Neural Network' self._model_type = 'BCNN' self.initializer = None self.activation = None self._last_layer_activation = None self.num_filters = None self.filter_len = None self.pool_length = None self.num_hidden = None self.reduce_lr_epsilon = None self.reduce_lr_min = None self.reduce_lr_patience = None self.l1 = None self.l2 = None self.maxnorm = None self.inv_model_precision = None # inverse model precision self.dropout_rate = 0.2 self.length_scale = 3 # prior length scale self.mc_num = 100 # increased to 100 due to high performance VI on GPU implemented on 14 April 2018 (Henry) self.val_size = 0.1 self.disable_dropout = False self.input_norm_mode = 1 self.labels_norm_mode = 2 self.keras_model_predict = None def pre_training_checklist_child(self, input_data, labels, input_err, labels_err): self.pre_training_checklist_master(input_data, labels) if isinstance(input_data, H5Loader): self.targetname = input_data.target input_data, labels = input_data.load() # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_input_err = input_err / self.input_std norm_labels_err = labels_err / self.labels_std if self.keras_model is None: # only compiler if there is no keras_model, e.g. fine-tuning does not required self.compile() self.train_idx, self.val_idx = train_test_split( np.arange(self.num_train + self.val_num), test_size=self.val_size) self.inv_model_precision = (2 * self.num_train * self.l2) / (self.length_scale**2 * (1 - self.dropout_rate)) self.training_generator = BayesianCNNDataGenerator( batch_size=self.batch_size, shuffle=True, steps_per_epoch=self.num_train // self.batch_size, data=[ norm_data[self.train_idx], norm_labels[self.train_idx], norm_input_err[self.train_idx], norm_labels_err[self.train_idx] ], manual_reset=False) val_batchsize = self.batch_size if len( self.val_idx) > self.batch_size else len(self.val_idx) self.validation_generator = BayesianCNNDataGenerator( batch_size=val_batchsize, shuffle=False, steps_per_epoch=max(self.val_num // self.batch_size, 1), data=[ norm_data[self.val_idx], norm_labels[self.val_idx], norm_input_err[self.val_idx], norm_labels_err[self.val_idx] ], manual_reset=True) return norm_data, norm_labels, norm_input_err, norm_labels_err def compile(self, optimizer=None, loss=None, metrics=None, weighted_metrics=None, loss_weights=None, sample_weight_mode=None): if optimizer is not None: self.optimizer = optimizer elif self.optimizer is None or self.optimizer == 'adam': self.optimizer = Adam(lr=self.lr, beta_1=self.beta_1, beta_2=self.beta_2, epsilon=self.optimizer_epsilon, decay=0.0) if metrics is not None: self.metrics = metrics if self.task == 'regression': if self._last_layer_activation is None: self._last_layer_activation = 'linear' elif self.task == 'classification': if self._last_layer_activation is None: self._last_layer_activation = 'softmax' elif self.task == 'binary_classification': if self._last_layer_activation is None: self._last_layer_activation = 'sigmoid' else: raise RuntimeError( 'Only "regression", "classification" and "binary_classification" are supported' ) self.keras_model, self.keras_model_predict, output_loss, variance_loss = self.model( ) if self.task == 'regression': self.metrics = [mean_absolute_error, mean_error ] if not self.metrics else self.metrics self.keras_model.compile(loss={ 'output': output_loss, 'variance_output': variance_loss }, optimizer=self.optimizer, metrics={'output': self.metrics}, weighted_metrics=weighted_metrics, loss_weights={ 'output': .5, 'variance_output': .5 } if not loss_weights else loss_weights, sample_weight_mode=sample_weight_mode) elif self.task == 'classification': self.metrics = [categorical_accuracy ] if not self.metrics else self.metrics self.keras_model.compile(loss={ 'output': output_loss, 'variance_output': variance_loss }, optimizer=self.optimizer, metrics={'output': self.metrics}, weighted_metrics=weighted_metrics, loss_weights={ 'output': .5, 'variance_output': .5 } if not loss_weights else loss_weights, sample_weight_mode=sample_weight_mode) elif self.task == 'binary_classification': self.metrics = [binary_accuracy ] if not self.metrics else self.metrics self.keras_model.compile(loss={ 'output': output_loss, 'variance_output': variance_loss }, optimizer=self.optimizer, metrics={'output': self.metrics}, weighted_metrics=weighted_metrics, loss_weights={ 'output': .5, 'variance_output': .5 } if not loss_weights else loss_weights, sample_weight_mode=sample_weight_mode) return None def train(self, input_data, labels, inputs_err=None, labels_err=None): """ Train a Bayesian neural network :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :return: None :rtype: NoneType :History: | 2018-Jan-06 - Written - Henry Leung (University of Toronto) | 2018-Apr-12 - Updated - Henry Leung (University of Toronto) """ if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) # Call the checklist to create astroNN folder and save parameters self.pre_training_checklist_child(input_data, labels, inputs_err, labels_err) reduce_lr = ReduceLROnPlateau(monitor='val_output_loss', factor=0.5, min_delta=self.reduce_lr_epsilon, patience=self.reduce_lr_patience, min_lr=self.reduce_lr_min, mode='min', verbose=2) self.virtual_cvslogger = VirutalCSVLogger() self.__callbacks = [reduce_lr, self.virtual_cvslogger ] # default must have unchangeable callbacks if self.callbacks is not None: if isinstance(self.callbacks, list): self.__callbacks.extend(self.callbacks) else: self.__callbacks.append(self.callbacks) start_time = time.time() self.history = self.keras_model.fit_generator( generator=self.training_generator, validation_data=self.validation_generator, epochs=self.max_epochs, verbose=self.verbose, workers=os.cpu_count(), callbacks=self.__callbacks, use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training, {(time.time() - start_time):.{2}f}s in total' ) if self.autosave is True: # Call the post training checklist to save parameters self.save() return None def train_on_batch(self, input_data, labels, inputs_err=None, labels_err=None): """ Train a Bayesian neural network by running a single gradient update on all of your data, suitable for fine-tuning :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :return: None :rtype: NoneType :History: | 2018-Aug-25 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_input_err = inputs_err / self.input_std norm_labels_err = labels_err / self.labels_std start_time = time.time() fit_generator = BayesianCNNDataGenerator( batch_size=input_data.shape[0], shuffle=False, steps_per_epoch=1, data=[norm_data, norm_labels, norm_input_err, norm_labels_err]) score = self.keras_model.fit_generator( fit_generator, epochs=1, verbose=self.verbose, workers=os.cpu_count(), use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training on Batch, {(time.time() - start_time):.{2}f}s in total' ) return None def post_training_checklist_child(self): self.keras_model.save(self.fullfilepath + _astroNN_MODEL_NAME) print(_astroNN_MODEL_NAME + f' saved to {(self.fullfilepath + _astroNN_MODEL_NAME)}') self.hyper_txt.write(f"Dropout Rate: {self.dropout_rate} \n") self.hyper_txt.flush() self.hyper_txt.close() data = { 'id': self.__class__.__name__ if self._model_identifier is None else self._model_identifier, 'pool_length': self.pool_length, 'filterlen': self.filter_len, 'filternum': self.num_filters, 'hidden': self.num_hidden, 'input': self._input_shape, 'labels': self._labels_shape, 'task': self.task, 'last_layer_activation': self._last_layer_activation, 'activation': self.activation, 'input_mean': self.input_mean.tolist(), 'inv_tau': self.inv_model_precision, 'length_scale': self.length_scale, 'labels_mean': self.labels_mean.tolist(), 'input_std': self.input_std.tolist(), 'labels_std': self.labels_std.tolist(), 'valsize': self.val_size, 'targetname': self.targetname, 'dropout_rate': self.dropout_rate, 'l1': self.l1, 'l2': self.l2, 'maxnorm': self.maxnorm, 'input_norm_mode': self.input_norm_mode, 'labels_norm_mode': self.labels_norm_mode, 'batch_size': self.batch_size } with open(self.fullfilepath + '/astroNN_model_parameter.json', 'w') as f: json.dump(data, f, indent=4, sort_keys=True) def test(self, input_data, inputs_err=None): """ Test model, High performance version designed for fast variational inference on GPU :param input_data: Data to be inferred with neural network :type input_data: ndarray :param inputs_err: Error for input_data, same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :return: prediction and prediction uncertainty :History: | 2018-Jan-06 - Written - Henry Leung (University of Toronto) | 2018-Apr-12 - Updated - Henry Leung (University of Toronto) """ self.has_model_check() if gpu_availability() is False and self.mc_num > 25: warnings.warn( f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can ' f'potentially be very slow! \n ' f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n' f'This is just a warning, and will not shown if mc_num < 25 on CPU' ) if self.mc_num < 2: raise AttributeError("mc_num cannot be smaller than 2") self.pre_testing_checklist_master() input_data = np.atleast_2d(input_data) if self.input_normalizer is not None: input_array = self.input_normalizer.normalize(input_data, calc=False) else: # Prevent shallow copy issue input_array = np.array(input_data) input_array -= self.input_mean input_array /= self.input_std # if no error array then just zeros if inputs_err is None: inputs_err = np.zeros_like(input_data) else: inputs_err = np.atleast_2d(inputs_err) inputs_err /= self.input_std total_test_num = input_data.shape[0] # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: batch_size = total_test_num else: batch_size = self.batch_size # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // batch_size) * batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator start_time = time.time() print("Starting Dropout Variational Inference") # Data Generator for prediction prediction_generator = BayesianCNNPredDataGenerator( batch_size=batch_size, shuffle=False, steps_per_epoch=data_gen_shape // batch_size, data=[input_array[:data_gen_shape], inputs_err[:data_gen_shape]]) new = FastMCInference(self.mc_num)(self.keras_model_predict) result = np.asarray(new.predict_generator(prediction_generator)) if remainder_shape != 0: # deal with remainder remainder_generator = BayesianCNNPredDataGenerator( batch_size=remainder_shape, shuffle=False, steps_per_epoch=1, data=[ input_array[data_gen_shape:], inputs_err[data_gen_shape:] ]) remainder_result = np.asarray( new.predict_generator(remainder_generator)) if remainder_shape == 1: remainder_result = np.expand_dims(remainder_result, axis=0) result = np.concatenate((result, remainder_result)) # in case only 1 test data point, in such case we need to add a dimension if result.ndim < 3 and batch_size == 1: result = np.expand_dims(result, axis=0) half_first_dim = result.shape[ 1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong predictions = result[:, :half_first_dim, 0] # mean prediction mc_dropout_uncertainty = result[:, :half_first_dim, 1] * ( self.labels_std**2) # model uncertainty predictions_var = np.exp(result[:, half_first_dim:, 0]) * ( self.labels_std**2) # predictive uncertainty print( f'Completed Dropout Variational Inference with {self.mc_num} forward passes, ' f'{(time.time() - start_time):.{2}f}s elapsed') if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize(predictions) else: predictions *= self.labels_std predictions += self.labels_mean if self.task == 'regression': # Predictive variance pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty pred_uncertainty = np.sqrt(pred_var) # Convert back to std error # final correction from variance to standard derivation mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty) predictive_uncertainty = np.sqrt(predictions_var) elif self.task == 'classification': # we want entropy for classification uncertainty predicted_class = np.argmax(predictions, axis=1) mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float) predictive_uncertainty = np.ones_like(predicted_class, dtype=float) # center variance predictions_var -= 1. for i in range(predicted_class.shape[0]): all_prediction = np.array(predictions[i, :]) mc_dropout_uncertainty[i] = -np.sum( all_prediction * np.log(all_prediction)) predictive_uncertainty[i] = predictions_var[i, predicted_class[i]] pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty # We only want the predicted class back predictions = predicted_class elif self.task == 'binary_classification': # we want entropy for classification uncertainty, so need prediction in logits space mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions), axis=0) # need to activate before round to int so that the prediction is always 0 or 1 predictions = np.rint(sigmoid(predictions)) predictive_uncertainty = predictions_var pred_uncertainty = mc_dropout_uncertainty + predictions_var else: raise AttributeError('Unknown Task') return predictions, { 'total': pred_uncertainty, 'model': mc_dropout_uncertainty, 'predictive': predictive_uncertainty } @deprecated def test_old(self, input_data, inputs_err=None): """ Tests model, it is recommended to use the new test() instead of this deprecated method :param input_data: Data to be inferred with neural network :type input_data: ndarray :param inputs_err: Error for input_data, same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :return: prediction and prediction uncertainty :History: 2018-Jan-06 - Written - Henry Leung (University of Toronto) """ self.pre_testing_checklist_master() if self.input_normalizer is not None: input_array = self.input_normalizer.normalize(input_data, calc=False) else: # Prevent shallow copy issue input_array = np.array(input_data) input_array -= self.input_mean input_array /= self.input_std # if no error array then just zeros if inputs_err is None: inputs_err = np.zeros_like(input_data) else: inputs_err /= self.input_std total_test_num = input_data.shape[0] # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: self.batch_size = total_test_num predictions = np.zeros( (self.mc_num, total_test_num, self._labels_shape)) predictions_var = np.zeros( (self.mc_num, total_test_num, self._labels_shape)) # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // self.batch_size) * self.batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator start_time = time.time() print("Starting Dropout Variational Inference") for i in range(self.mc_num): if i % 5 == 0: print( f'Completed {i} of {self.mc_num} Monte Carlo Dropout, {(time.time() - start_time):.{2}f}s ' f'elapsed') # Data Generator for prediction prediction_generator = BayesianCNNPredDataGenerator( batch_size=self.batch_size, shuffle=False, steps_per_epoch=data_gen_shape // self.batch_size, data=[ input_array[:data_gen_shape], inputs_err[:data_gen_shape] ]) result = np.asarray( self.keras_model_predict.predict_generator( prediction_generator)) if result.ndim < 2: # in case only 1 test data point, in such case we need to add a dimension result = np.expand_dims(result, axis=0) half_first_dim = result.shape[ 1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong predictions[ i, :data_gen_shape] = result[:, :half_first_dim].reshape( (data_gen_shape, self._labels_shape)) predictions_var[ i, :data_gen_shape] = result[:, half_first_dim:].reshape( (data_gen_shape, self._labels_shape)) if remainder_shape != 0: remainder_data = input_array[data_gen_shape:] remainder_data_err = inputs_err[data_gen_shape:] # assume its caused by mono images, so need to expand dim by 1 if len(input_array[0].shape) != len(self._input_shape): remainder_data = np.expand_dims(remainder_data, axis=-1) remainder_data_err = np.expand_dims(remainder_data_err, axis=-1) result = self.keras_model_predict.predict({ 'input': remainder_data, 'input_err': remainder_data_err }) predictions[ i, data_gen_shape:] = result[:, :half_first_dim].reshape( (remainder_shape, self._labels_shape)) predictions_var[ i, data_gen_shape:] = result[:, half_first_dim:].reshape( (remainder_shape, self._labels_shape)) print( f'Completed Dropout Variational Inference, {(time.time() - start_time):.{2}f}s in total' ) if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize(predictions) else: predictions *= self.labels_std predictions += self.labels_mean pred = np.mean(predictions, axis=0) if self.task == 'regression': # Predictive variance mc_dropout_uncertainty = np.var(predictions, axis=0) # var predictive_uncertainty = np.mean(np.exp(predictions_var) * (np.array(self.labels_std)**2), axis=0) pred_var = predictive_uncertainty + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty pred_uncertainty = np.sqrt(pred_var) # Convert back to std error # final correction from variance to standard derivation mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty) predictive_uncertainty = np.sqrt(predictive_uncertainty) elif self.task == 'classification': # we want entropy for classification uncertainty pred_shape = pred.shape[0] pred = np.argmax(pred, axis=1) predictions_var = np.mean(predictions_var, axis=0) mc_dropout_uncertainty = np.ones_like(pred, dtype=float) predictive_uncertainty = np.ones_like(pred, dtype=float) for i in range(pred_shape): all_prediction = np.array(predictions[:, i, pred[i]]) mc_dropout_uncertainty[i] = -np.sum( all_prediction * np.log(all_prediction)) predictive_uncertainty[i] = np.array(predictions_var[i, pred[i]]) pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty elif self.task == 'binary_classification': # we want entropy for classification uncertainty mc_dropout_uncertainty = -np.sum( pred * np.log(pred), axis=0) # need to use raw prediction for uncertainty pred = np.rint(pred) predictive_uncertainty = np.mean(predictions_var, axis=0) pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty else: raise AttributeError('Unknown Task') return pred, { 'total': pred_uncertainty, 'model': mc_dropout_uncertainty, 'predictive': predictive_uncertainty } def evaluate(self, input_data, labels, inputs_err=None, labels_err=None): """ Evaluate neural network by provided input data and labels and get back a metrics score :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :return: metrics score dictionary :rtype: dict :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_input_err = inputs_err / self.input_std norm_labels_err = labels_err / self.labels_std eval_batchsize = self.batch_size if input_data.shape[ 0] > self.batch_size else input_data.shape[0] steps = input_data.shape[0] // self.batch_size if input_data.shape[ 0] > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") evaluate_generator = BayesianCNNDataGenerator( batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels, norm_input_err, norm_labels_err]) scores = self.keras_model.evaluate_generator(evaluate_generator) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = [] if isinstance(self.keras_model.metrics, dict): func_list = self.keras_model.metrics[outputname[0]] else: func_list = self.keras_model.metrics for func in func_list: if hasattr(func, __name__): funcname.append(func.__name__) else: funcname.append(func.__class__.__name__) # funcname = [func.__name__ for func in self.keras_model.metrics[outputname[0]]] loss_outputname = ['loss_' + name for name in outputname] output_funcname = [outputname[0] + '_' + name for name in funcname] list_names = ['loss', *loss_outputname, *output_funcname] print( f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed' ) return {name: score for name, score in zip(list_names, scores)}
def evaluate(self, input_data, labels, inputs_err=None, labels_err=None): """ Evaluate neural network by provided input data and labels and get back a metrics score :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :return: metrics score dictionary :rtype: dict :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) input_data = {"input": input_data} labels = {"output": labels} # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_input_err = inputs_err / self.input_std['input'] norm_labels_err = labels_err / self.labels_std['output'] norm_data.update({ "input_err": norm_input_err, "labels_err": norm_labels_err }) norm_labels.update({"variance_output": norm_labels["output"]}) total_num = input_data['input'].shape[0] eval_batchsize = self.batch_size if total_num > self.batch_size else total_num steps = total_num // self.batch_size if total_num > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") evaluate_generator = BayesianCNNDataGenerator( batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels]) scores = self.keras_model.evaluate(evaluate_generator) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = self.keras_model.metrics_names print( f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed' ) return list_to_dict(funcname, scores)
def load_folder(folder=None): """ To load astroNN model object from folder :param folder: [optional] you should provide folder name if outside folder, do not specific when you are inside the folder :type folder: str :return: astroNN Neural Network instance :rtype: astroNN.nn.NeuralNetMaster.NeuralNetMaster :History: 2017-Dec-29 - Written - Henry Leung (University of Toronto) """ currentdir = os.getcwd() if folder is not None: fullfilepath = os.path.join(currentdir, folder) else: fullfilepath = currentdir astronn_model_obj = None if folder is not None and os.path.isfile(os.path.join(folder, 'astroNN_model_parameter.json')) is True: with open(os.path.join(folder, 'astroNN_model_parameter.json')) as f: parameter = json.load(f) f.close() elif os.path.isfile('astroNN_model_parameter.json') is True: with open('astroNN_model_parameter.json') as f: parameter = json.load(f) f.close() elif folder is not None and not os.path.exists(folder): raise IOError('Folder not exists: ' + str(currentdir + '/' + folder)) else: raise FileNotFoundError('Are you sure this is an astroNN generated folder? Or it is a folder trained by old ' 'astroNN version?') identifier = parameter['id'] if identifier == 'ApogeeCNN': astronn_model_obj = ApogeeCNN() elif identifier == 'ApogeeBCNN': astronn_model_obj = ApogeeBCNN() elif identifier == 'ApogeeBCNNCensored': astronn_model_obj = ApogeeBCNNCensored() elif identifier == 'ApogeeCVAE': astronn_model_obj = ApogeeCVAE() elif identifier == 'Cifar10CNN': astronn_model_obj = Cifar10CNN() elif identifier == 'MNIST_BCNN': astronn_model_obj = MNIST_BCNN() elif identifier == 'Galaxy10CNN': astronn_model_obj = Galaxy10CNN() elif identifier == 'StarNet2017': astronn_model_obj = StarNet2017() elif identifier == 'GalaxyGAN2017': astronn_model_obj = GalaxyGAN2017() elif identifier == 'Galaxy10GAN': astronn_model_obj = Galaxy10GAN() else: unknown_model_message = f'Unknown model identifier -> {identifier}!' # try to load custom model from CUSTOM_MODEL_PATH CUSTOM_MODEL_PATH = custom_model_path_reader() # try the current folder and see if there is any .py on top of CUSTOM_MODEL_PATH list_py_files = [os.path.join(fullfilepath, f) for f in os.listdir(fullfilepath) if f.endswith(".py")] if CUSTOM_MODEL_PATH is None and list_py_files is None: print("\n") raise TypeError(unknown_model_message) else: import sys from importlib import import_module for path_list in (path_list for path_list in [CUSTOM_MODEL_PATH, list_py_files] if path_list is not None): for path in path_list: head, tail = os.path.split(path) sys.path.insert(0, head) try: model = getattr(import_module(tail.strip('.py')), str(identifier)) astronn_model_obj = model() except AttributeError: pass if astronn_model_obj is None: print("\n") raise TypeError(unknown_model_message) astronn_model_obj.currentdir = currentdir astronn_model_obj.fullfilepath = fullfilepath astronn_model_obj.folder_name = folder if folder is not None else os.path.basename(os.path.normpath(currentdir)) # Must have parameter astronn_model_obj._input_shape = parameter['input'] astronn_model_obj._labels_shape = parameter['labels'] astronn_model_obj.num_hidden = parameter['hidden'] astronn_model_obj.input_norm_mode = parameter['input_norm_mode'] astronn_model_obj.labels_norm_mode = parameter['labels_norm_mode'] astronn_model_obj.input_mean = np.array(parameter['input_mean']) astronn_model_obj.labels_mean = np.array(parameter['labels_mean']) astronn_model_obj.input_std = np.array(parameter['input_std']) astronn_model_obj.labels_std = np.array(parameter['labels_std']) astronn_model_obj.batch_size = parameter['batch_size'] astronn_model_obj.targetname = parameter['targetname'] astronn_model_obj.val_size = parameter['valsize'] # create normalizer and set correct mean and std astronn_model_obj.input_normalizer = Normalizer(mode=astronn_model_obj.input_norm_mode) astronn_model_obj.labels_normalizer = Normalizer(mode=astronn_model_obj.labels_norm_mode) astronn_model_obj.input_normalizer.mean_labels = astronn_model_obj.input_mean astronn_model_obj.input_normalizer.std_labels = astronn_model_obj.input_std astronn_model_obj.labels_normalizer.mean_labels = astronn_model_obj.labels_mean astronn_model_obj.labels_normalizer.std_labels = astronn_model_obj.labels_std # Conditional parameter depends on neural net architecture try: astronn_model_obj.num_filters = parameter['filternum'] except KeyError: pass try: astronn_model_obj.filter_len = parameter['filterlen'] except KeyError: pass try: pool_length = parameter['pool_length'] if isinstance(pool_length, int): # multi-dimensional case astronn_model_obj.pool_length = parameter['pool_length'] else: astronn_model_obj.pool_length = list(parameter['pool_length']) except KeyError: pass try: # need to convert to int because of keras do not want array or list astronn_model_obj.latent_dim = int(parameter['latent']) except KeyError: pass try: astronn_model_obj.task = parameter['task'] except KeyError: pass try: astronn_model_obj.dropout_rate = parameter['dropout_rate'] except KeyError: pass try: # if inverse model precision exists, so does length_scale astronn_model_obj.inv_model_precision = parameter['inv_tau'] astronn_model_obj.length_scale = parameter['length_scale'] except KeyError: pass try: astronn_model_obj.l1 = parameter['l1'] except KeyError: pass try: astronn_model_obj.l2 = parameter['l2'] except KeyError: pass try: astronn_model_obj.maxnorm = parameter['maxnorm'] except KeyError: pass with h5py.File(os.path.join(astronn_model_obj.fullfilepath, 'model_weights.h5'), mode='r') as f: training_config = f.attrs.get('training_config') training_config = json.loads(training_config.decode('utf-8')) optimizer_config = training_config['optimizer_config'] optimizer = optimizers.deserialize(optimizer_config) # Recover loss functions and metrics. losses = convert_custom_objects(training_config['loss']) try: try: [losses_lookup(losses[loss]) for loss in losses] except TypeError: losses_lookup(losses) except: pass metrics = convert_custom_objects(training_config['metrics']) # its weird that keras needs -> metrics[metric][0] instead of metrics[metric] likes losses, need attention try: try: [losses_lookup(metrics[metric][0]) for metric in metrics] except TypeError: losses_lookup(metrics[0]) except: pass sample_weight_mode = training_config['sample_weight_mode'] loss_weights = training_config['loss_weights'] # compile the model astronn_model_obj.compile(optimizer=optimizer) # set weights astronn_model_obj.keras_model.load_weights(os.path.join(astronn_model_obj.fullfilepath, 'model_weights.h5')) # Build train function (to get weight updates), need to consider Sequential model too astronn_model_obj.keras_model._make_train_function() if isinstance(astronn_model_obj.keras_model, Sequential): astronn_model_obj.keras_model.model._make_train_function() else: astronn_model_obj.keras_model._make_train_function() optimizer_weights_group = f['optimizer_weights'] optimizer_weight_names = [n.decode('utf8') for n in optimizer_weights_group.attrs['weight_names']] optimizer_weight_values = [optimizer_weights_group[n] for n in optimizer_weight_names] astronn_model_obj.keras_model.optimizer.set_weights(optimizer_weight_values) print("========================================================") print(f"Loaded astroNN model, model type: {astronn_model_obj.name} -> {identifier}") print("========================================================") return astronn_model_obj
def evaluate(self, input_data, labels): """ Evaluate neural network by provided input data and labels and get back a metrics score :param input_data: Data to be inferred with neural network :type input_data: ndarray :param labels: labels :type labels: ndarray :return: metrics score dictionary :rtype: dict :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) eval_batchsize = self.batch_size if input_data.shape[ 0] > self.batch_size else input_data.shape[0] steps = input_data.shape[0] // self.batch_size if input_data.shape[ 0] > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") evaluate_generator = CNNDataGenerator(batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels]) scores = self.keras_model.evaluate_generator(evaluate_generator) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = [] if isinstance(self.keras_model.metrics, dict): func_list = self.keras_model.metrics[outputname[0]] else: func_list = self.keras_model.metrics for func in func_list: if hasattr(func, __name__): funcname.append(func.__name__) else: funcname.append(func.__class__.__name__) # funcname = [func.__name__ for func in self.keras_model.metrics] output_funcname = [outputname[0] + '_' + name for name in funcname] list_names = ['loss', *output_funcname] print( f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed' ) return {name: score for name, score in zip(list_names, scores)}
def pre_training_checklist_child(self, input_data, labels): input_data, labels = self.pre_training_checklist_master( input_data, labels) # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_data.update({ "input_err": (input_data['input_err'] / self.input_std['input']), "labels_err": input_data['labels_err'] / self.labels_std['output'] }) norm_labels.update({"variance_output": norm_labels['output']}) if self.keras_model is None: # only compile if there is no keras_model, e.g. fine-tuning does not required self.compile() self.train_idx, self.val_idx = train_test_split( np.arange(self.num_train + self.val_num), test_size=self.val_size) norm_data_training = {} norm_data_val = {} norm_labels_training = {} norm_labels_val = {} for name in norm_data.keys(): norm_data_training.update({name: norm_data[name][self.train_idx]}) norm_data_val.update({name: norm_data[name][self.val_idx]}) for name in norm_labels.keys(): norm_labels_training.update( {name: norm_labels[name][self.train_idx]}) norm_labels_val.update({name: norm_labels[name][self.val_idx]}) self.inv_model_precision = (2 * self.num_train * self.l2) / (self.length_scale**2 * (1 - self.dropout_rate)) self.training_generator = BayesianCNNDataGenerator( batch_size=self.batch_size, shuffle=True, steps_per_epoch=self.num_train // self.batch_size, data=[norm_data_training, norm_labels_training], manual_reset=False) val_batchsize = self.batch_size if len( self.val_idx) > self.batch_size else len(self.val_idx) self.validation_generator = BayesianCNNDataGenerator( batch_size=val_batchsize, shuffle=False, steps_per_epoch=max(self.val_num // self.batch_size, 1), data=[norm_data_val, norm_labels_val], manual_reset=True) return norm_data, norm_labels
class BayesianCNNBase(NeuralNetMaster, ABC): """ Top-level class for a Bayesian convolutional neural network :History: 2018-Jan-06 - Written - Henry Leung (University of Toronto) """ def __init__(self): super().__init__() self.name = 'Bayesian Convolutional Neural Network' self._model_type = 'BCNN' self.initializer = None self.activation = None self._last_layer_activation = None self.num_filters = None self.filter_len = None self.pool_length = None self.num_hidden = None self.reduce_lr_epsilon = None self.reduce_lr_min = None self.reduce_lr_patience = None self.l1 = None self.l2 = None self.maxnorm = None self.inv_model_precision = None # inverse model precision self.dropout_rate = 0.2 self.length_scale = 3 # prior length scale self.mc_num = 100 # increased to 100 due to high performance VI on GPU implemented on 14 April 2018 (Henry) self.val_size = 0.1 self.disable_dropout = False self.input_norm_mode = 1 self.labels_norm_mode = 2 self.keras_model_predict = None def pre_training_checklist_child(self, input_data, labels, sample_weights): input_data, labels = self.pre_training_checklist_master( input_data, labels) # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_data.update({ "input_err": (input_data['input_err'] / self.input_std['input']), "labels_err": input_data['labels_err'] / self.labels_std['output'] }) norm_labels.update({"variance_output": norm_labels['output']}) if self.keras_model is None: # only compile if there is no keras_model, e.g. fine-tuning does not required self.compile() self.train_idx, self.val_idx = train_test_split( np.arange(self.num_train + self.val_num), test_size=self.val_size) norm_data_training = {} norm_data_val = {} norm_labels_training = {} norm_labels_val = {} for name in norm_data.keys(): norm_data_training.update({name: norm_data[name][self.train_idx]}) norm_data_val.update({name: norm_data[name][self.val_idx]}) for name in norm_labels.keys(): norm_labels_training.update( {name: norm_labels[name][self.train_idx]}) norm_labels_val.update({name: norm_labels[name][self.val_idx]}) if sample_weights is not None: sample_weights_training = sample_weights[self.train_idx] sample_weights_val = sample_weights[self.val_idx] else: sample_weights_training = None sample_weights_val = None self.inv_model_precision = (2 * self.num_train * self.l2) / (self.length_scale**2 * (1 - self.dropout_rate)) self.training_generator = BayesianCNNDataGenerator( batch_size=self.batch_size, shuffle=True, steps_per_epoch=self.num_train // self.batch_size, data=[norm_data_training, norm_labels_training], manual_reset=False, sample_weights=sample_weights_training) val_batchsize = self.batch_size if len( self.val_idx) > self.batch_size else len(self.val_idx) self.validation_generator = BayesianCNNDataGenerator( batch_size=val_batchsize, shuffle=False, steps_per_epoch=max(self.val_num // self.batch_size, 1), data=[norm_data_val, norm_labels_val], manual_reset=True, sample_weights=sample_weights_val) return norm_data_training, norm_data_val, norm_labels_training, norm_labels_val, sample_weights_training, sample_weights_val def compile(self, optimizer=None, loss=None, metrics=None, weighted_metrics=None, loss_weights=None, sample_weight_mode=None): if optimizer is not None: self.optimizer = optimizer elif self.optimizer is None or self.optimizer == 'adam': self.optimizer = Adam(learning_rate=self.lr, beta_1=self.beta_1, beta_2=self.beta_2, epsilon=self.optimizer_epsilon, decay=0.0) if metrics is not None: self.metrics = metrics if self.task == 'regression': if self._last_layer_activation is None: self._last_layer_activation = 'linear' elif self.task == 'classification': if self._last_layer_activation is None: self._last_layer_activation = 'softmax' elif self.task == 'binary_classification': if self._last_layer_activation is None: self._last_layer_activation = 'sigmoid' else: raise RuntimeError( 'Only "regression", "classification" and "binary_classification" are supported' ) self.keras_model, self.keras_model_predict, self.output_loss, self.variance_loss = self.model( ) if self.task == 'regression': self._output_loss = lambda predictive, labelerr: mse_lin_wrapper( predictive, labelerr) elif self.task == 'classification': self._output_loss = lambda predictive, labelerr: bayesian_categorical_crossentropy_wrapper( predictive) elif self.task == 'binary_classification': self._output_loss = lambda predictive, labelerr: bayesian_binary_crossentropy_wrapper( predictive) else: raise RuntimeError( 'Only "regression", "classification" and "binary_classification" are supported' ) # all zero losss as dummy lose if self.task == 'regression': self.metrics = [mean_absolute_error, mean_error ] if not self.metrics else self.metrics self.keras_model.compile(optimizer=self.optimizer, loss=zeros_loss, metrics=self.metrics, weighted_metrics=weighted_metrics, sample_weight_mode=sample_weight_mode) elif self.task == 'classification': self.metrics = [categorical_accuracy ] if not self.metrics else self.metrics self.keras_model.compile(optimizer=self.optimizer, loss=zeros_loss, metrics={'output': self.metrics}, weighted_metrics=weighted_metrics, sample_weight_mode=sample_weight_mode) elif self.task == 'binary_classification': self.metrics = [binary_accuracy ] if not self.metrics else self.metrics self.keras_model.compile(optimizer=self.optimizer, loss=zeros_loss, metrics={'output': self.metrics}, weighted_metrics=weighted_metrics, sample_weight_mode=sample_weight_mode) # inject custom training step if needed try: self.custom_train_step() except NotImplementedError: pass except TypeError: self.keras_model.train_step = self.custom_train_step # inject custom testing step if needed try: self.custom_test_step() except NotImplementedError: pass except TypeError: self.keras_model.test_step = self.custom_test_step return None def custom_train_step(self, data): """ Custom training logic :param data: :return: """ data = data_adapter.expand_1d(data) x, y, sample_weight = data_adapter.unpack_x_y_sample_weight(data) # Run forward pass. with tf.GradientTape() as tape: y_pred = self.keras_model(x, training=True) self.keras_model.compiled_loss._losses = self._output_loss( y_pred[1], x['labels_err']) self.keras_model.compiled_loss._losses = nest.map_structure( self.keras_model.compiled_loss._get_loss_object, self.keras_model.compiled_loss._losses) self.keras_model.compiled_loss._losses = nest.flatten( self.keras_model.compiled_loss._losses) loss = self.keras_model.compiled_loss( y, y_pred, sample_weight, regularization_losses=self.keras_model.losses) # Run backwards pass. self.keras_model.optimizer.minimize( loss, self.keras_model.trainable_variables, tape=tape) self.keras_model.compiled_metrics.update_state(y, y_pred, sample_weight) # Collect metrics to return return_metrics = {} for metric in self.keras_model.metrics: result = metric.result() if isinstance(result, dict): return_metrics.update(result) else: return_metrics[metric.name] = result return return_metrics def custom_test_step(self, data): data = data_adapter.expand_1d(data) x, y, sample_weight = data_adapter.unpack_x_y_sample_weight(data) y_pred = self.keras_model(x, training=False) # Updates stateful loss metrics. temploss = self._output_loss(y_pred[1], x['labels_err']) self.keras_model.compiled_loss._losses = temploss self.keras_model.compiled_loss._losses = nest.map_structure( self.keras_model.compiled_loss._get_loss_object, self.keras_model.compiled_loss._losses) self.keras_model.compiled_loss._losses = nest.flatten( self.keras_model.compiled_loss._losses) self.keras_model.compiled_loss( y, y_pred, sample_weight, regularization_losses=self.keras_model.losses) self.keras_model.compiled_metrics.update_state(y, y_pred, sample_weight) # Collect metrics to return return_metrics = {} for metric in self.keras_model.metrics: result = metric.result() if isinstance(result, dict): return_metrics.update(result) else: return_metrics[metric.name] = result return return_metrics def fit(self, input_data, labels, inputs_err=None, labels_err=None, sample_weights=None, experimental=False): """ Train a Bayesian neural network :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :param sample_weights: Sample weights (if any) :type sample_weights: Union([NoneType, ndarray]) :return: None :rtype: NoneType :History: | 2018-Jan-06 - Written - Henry Leung (University of Toronto) | 2018-Apr-12 - Updated - Henry Leung (University of Toronto) """ if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) # TODO: allow named inputs too?? input_data = { "input": input_data, "input_err": inputs_err, "labels_err": labels_err } labels = {"output": labels, "variance_output": labels} # Call the checklist to create astroNN folder and save parameters norm_data_training, norm_data_val, norm_labels_training, norm_labels_val, sample_weights_training, sample_weights_val = self.pre_training_checklist_child( input_data, labels, sample_weights) # norm_data_training['labels_err'] = norm_data_training['labels_err'].filled(MAGIC_NUMBER).astype(np.float32) # TODO: fix the monitor name reduce_lr = ReduceLROnPlateau(monitor='val_output_mean_absolute_error', factor=0.5, min_delta=self.reduce_lr_epsilon, patience=self.reduce_lr_patience, min_lr=self.reduce_lr_min, mode='min', verbose=2) self.virtual_cvslogger = VirutalCSVLogger() self.__callbacks = [reduce_lr, self.virtual_cvslogger ] # default must have unchangeable callbacks if self.callbacks is not None: if isinstance(self.callbacks, list): self.__callbacks.extend(self.callbacks) else: self.__callbacks.append(self.callbacks) start_time = time.time() if experimental: dataset = tf.data.Dataset.from_tensor_slices( (norm_data_training, norm_labels_training, sample_weights_training)).batch(self.batch_size).shuffle( 5000, reshuffle_each_iteration=True).prefetch(tf.data.AUTOTUNE) val_dataset = tf.data.Dataset.from_tensor_slices( (norm_data_val, norm_labels_val, sample_weights_val)).batch( self.batch_size).prefetch(tf.data.AUTOTUNE) self.history = self.keras_model.fit( dataset, validation_data=val_dataset, epochs=self.max_epochs, verbose=self.verbose, workers=os.cpu_count() // 2, callbacks=self.__callbacks, use_multiprocessing=MULTIPROCESS_FLAG) else: self.history = self.keras_model.fit( self.training_generator, validation_data=self.validation_generator, epochs=self.max_epochs, verbose=self.verbose, workers=os.cpu_count() // 2, callbacks=self.__callbacks, use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training, {(time.time() - start_time):.{2}f}s in total' ) if self.autosave is True: # Call the post training checklist to save parameters self.save() return None def fit_on_batch(self, input_data, labels, inputs_err=None, labels_err=None, sample_weights=None): """ Train a Bayesian neural network by running a single gradient update on all of your data, suitable for fine-tuning :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :param sample_weights: Sample weights (if any) :type sample_weights: Union([NoneType, ndarray]) :return: None :rtype: NoneType :History: | 2018-Aug-25 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) input_data = { "input": input_data, "input_err": inputs_err, "labels_err": labels_err } labels = {"output": labels, "variance_output": labels} # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_data.update({ "input_err": (input_data['input_err'] / self.input_std['input']), "labels_err": input_data['labels_err'] / self.labels_std['output'] }) norm_labels.update({"variance_output": norm_labels['output']}) start_time = time.time() fit_generator = BayesianCNNDataGenerator( batch_size=input_data['input'].shape[0], shuffle=False, steps_per_epoch=1, data=[norm_data, norm_labels], sample_weights=sample_weights) score = self.keras_model.fit(fit_generator, epochs=1, verbose=self.verbose, workers=os.cpu_count(), use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training on Batch, {(time.time() - start_time):.{2}f}s in total' ) return None def post_training_checklist_child(self): self.keras_model.save(self.fullfilepath + _astroNN_MODEL_NAME) print(_astroNN_MODEL_NAME + f' saved to {(self.fullfilepath + _astroNN_MODEL_NAME)}') self.hyper_txt.write(f"Dropout Rate: {self.dropout_rate} \n") self.hyper_txt.flush() self.hyper_txt.close() data = { 'id': self.__class__.__name__ if self._model_identifier is None else self._model_identifier, 'pool_length': self.pool_length, 'filterlen': self.filter_len, 'filternum': self.num_filters, 'hidden': self.num_hidden, 'input': self._input_shape, 'labels': self._labels_shape, 'task': self.task, 'last_layer_activation': self._last_layer_activation, 'activation': self.activation, 'input_mean': dict_np_to_dict_list(self.input_mean), 'inv_tau': self.inv_model_precision, 'length_scale': self.length_scale, 'labels_mean': dict_np_to_dict_list(self.labels_mean), 'input_std': dict_np_to_dict_list(self.input_std), 'labels_std': dict_np_to_dict_list(self.labels_std), 'valsize': self.val_size, 'targetname': self.targetname, 'dropout_rate': self.dropout_rate, 'l1': self.l1, 'l2': self.l2, 'maxnorm': self.maxnorm, 'input_norm_mode': self.input_normalizer.normalization_mode, 'labels_norm_mode': self.labels_normalizer.normalization_mode, 'input_names': self.input_names, 'output_names': self.output_names, 'batch_size': self.batch_size } with open(self.fullfilepath + '/astroNN_model_parameter.json', 'w') as f: json.dump(data, f, indent=4, sort_keys=True) def predict(self, input_data, inputs_err=None): """ Test model, High performance version designed for fast variational inference on GPU :param input_data: Data to be inferred with neural network :type input_data: ndarray :param inputs_err: Error for input_data, same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :return: prediction and prediction uncertainty :History: | 2018-Jan-06 - Written - Henry Leung (University of Toronto) | 2018-Apr-12 - Updated - Henry Leung (University of Toronto) """ self.has_model_check() if gpu_availability() is False and self.mc_num > 25: warnings.warn( f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can ' f'potentially be very slow! \n ' f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n' f'This is just a warning, and will not shown if mc_num < 25 on CPU' ) if self.mc_num < 2: raise AttributeError("mc_num cannot be smaller than 2") # if no error array then just zeros if inputs_err is None: inputs_err = np.zeros_like(input_data) else: inputs_err = np.atleast_2d(inputs_err) inputs_err /= self.input_std['input'] input_data = {"input": input_data, "input_err": inputs_err} input_data = self.pre_testing_checklist_master(input_data) if self.input_normalizer is not None: input_array = self.input_normalizer.normalize(input_data, calc=False) else: # Prevent shallow copy issue input_array = np.array(input_data) input_array -= self.input_mean['input'] input_array /= self.input_std['input'] total_test_num = input_data['input'].shape[0] # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: batch_size = total_test_num else: batch_size = self.batch_size # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // batch_size) * batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator norm_data_main = {} norm_data_remainder = {} for name in input_array.keys(): norm_data_main.update({name: input_array[name][:data_gen_shape]}) norm_data_remainder.update( {name: input_array[name][data_gen_shape:]}) # Data Generator for prediction with tqdm(total=total_test_num, unit="sample") as pbar: pbar.set_postfix({'Monte-Carlo': self.mc_num}) # suppress pfor warning from TF old_level = tf.get_logger().level tf.get_logger().setLevel('ERROR') prediction_generator = BayesianCNNPredDataGenerator( batch_size=batch_size, shuffle=False, steps_per_epoch=data_gen_shape // batch_size, data=[norm_data_main], pbar=pbar) new = FastMCInference(self.mc_num)(self.keras_model_predict) result = np.asarray(new.predict(prediction_generator)) if remainder_shape != 0: # deal with remainder remainder_generator = BayesianCNNPredDataGenerator( batch_size=remainder_shape, shuffle=False, steps_per_epoch=1, data=[norm_data_remainder], pbar=pbar) remainder_result = np.asarray(new.predict(remainder_generator)) if remainder_shape == 1: remainder_result = np.expand_dims(remainder_result, axis=0) result = np.concatenate((result, remainder_result)) tf.get_logger().setLevel(old_level) # in case only 1 test data point, in such case we need to add a dimension if result.ndim < 3 and batch_size == 1: result = np.expand_dims(result, axis=0) half_first_dim = result.shape[ 1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong predictions = result[:, :half_first_dim, 0] # mean prediction mc_dropout_uncertainty = result[:, :half_first_dim, 1] * ( self.labels_std['output']**2) # model uncertainty predictions_var = np.exp(result[:, half_first_dim:, 0]) * ( self.labels_std['output']**2) # predictive uncertainty if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize( list_to_dict([self.keras_model.output_names[0]], predictions)) predictions = predictions['output'] else: predictions *= self.labels_std['output'] predictions += self.labels_mean['output'] if self.task == 'regression': # Predictive variance pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty pred_uncertainty = np.sqrt(pred_var) # Convert back to std error # final correction from variance to standard derivation mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty) predictive_uncertainty = np.sqrt(predictions_var) elif self.task == 'classification': # we want entropy for classification uncertainty predicted_class = np.argmax(predictions, axis=1) mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float) predictive_uncertainty = np.ones_like(predicted_class, dtype=float) # center variance predictions_var -= 1. for i in range(predicted_class.shape[0]): all_prediction = np.array(predictions[i, :]) mc_dropout_uncertainty[i] = -np.sum( all_prediction * np.log(all_prediction)) predictive_uncertainty[i] = predictions_var[i, predicted_class[i]] pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty # We only want the predicted class back predictions = predicted_class elif self.task == 'binary_classification': # we want entropy for classification uncertainty, so need prediction in logits space mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions), axis=0) # need to activate before round to int so that the prediction is always 0 or 1 predictions = np.rint(sigmoid(predictions)) predictive_uncertainty = predictions_var pred_uncertainty = mc_dropout_uncertainty + predictions_var else: raise AttributeError('Unknown Task') return predictions, { 'total': pred_uncertainty, 'model': mc_dropout_uncertainty, 'predictive': predictive_uncertainty } def predict_dataset(self, file): class BayesianCNNPredDataGeneratorV2(GeneratorMaster): def __init__(self, batch_size, shuffle, steps_per_epoch, manual_reset=False, pbar=None, nn_model=None): super().__init__(batch_size=batch_size, shuffle=shuffle, steps_per_epoch=steps_per_epoch, data=None, manual_reset=manual_reset) self.pbar = pbar # initial idx self.idx_list = self._get_exploration_order(range(len(file))) self.current_idx = 0 self.nn_model = nn_model def _data_generation(self, idx_list_temp): # Generate data inputs = self.nn_model.input_normalizer.normalize( { "input": file[idx_list_temp], "input_err": np.zeros_like(file[idx_list_temp]) }, calc=False) x = self.input_d_checking(inputs, np.arange(len(idx_list_temp))) return x def __getitem__(self, index): x = self._data_generation( self.idx_list[index * self.batch_size:(index + 1) * self.batch_size]) if self.pbar: self.pbar.update(self.batch_size) return x def on_epoch_end(self): # shuffle the list when epoch ends for the next epoch self.idx_list = self._get_exploration_order(range(len(file))) self.has_model_check() if gpu_availability() is False and self.mc_num > 25: warnings.warn( f'You are using CPU version Tensorflow, doing {self.mc_num} times Monte Carlo Inference can ' f'potentially be very slow! \n ' f'A possible fix is to decrease the mc_num parameter of the model to do less MC Inference \n' f'This is just a warning, and will not shown if mc_num < 25 on CPU' ) if self.mc_num < 2: raise AttributeError("mc_num cannot be smaller than 2") total_test_num = len(file) # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: batch_size = total_test_num else: batch_size = self.batch_size # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // batch_size) * batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator # Data Generator for prediction with tqdm(total=total_test_num, unit="sample") as pbar: pbar.set_postfix({'Monte-Carlo': self.mc_num}) # suppress pfor warning from TF old_level = tf.get_logger().level tf.get_logger().setLevel('ERROR') prediction_generator = BayesianCNNPredDataGeneratorV2( batch_size=batch_size, shuffle=False, steps_per_epoch=data_gen_shape // batch_size, pbar=pbar, nn_model=self) new = FastMCInference(self.mc_num)(self.keras_model_predict) result = np.asarray(new.predict(prediction_generator)) if remainder_shape != 0: # deal with remainder remainder_generator = BayesianCNNPredDataGeneratorV2( batch_size=remainder_shape, shuffle=False, steps_per_epoch=1, pbar=pbar, nn_model=self) remainder_result = np.asarray(new.predict(remainder_generator)) if remainder_shape == 1: remainder_result = np.expand_dims(remainder_result, axis=0) result = np.concatenate((result, remainder_result)) tf.get_logger().setLevel(old_level) # in case only 1 test data point, in such case we need to add a dimension if result.ndim < 3 and batch_size == 1: result = np.expand_dims(result, axis=0) half_first_dim = result.shape[ 1] // 2 # result.shape[1] is guarantee an even number, otherwise sth is wrong predictions = result[:, :half_first_dim, 0] # mean prediction mc_dropout_uncertainty = result[:, :half_first_dim, 1] * ( self.labels_std['output']**2) # model uncertainty predictions_var = np.exp(result[:, half_first_dim:, 0]) * ( self.labels_std['output']**2) # predictive uncertainty if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize( list_to_dict([self.keras_model.output_names[0]], predictions)) predictions = predictions['output'] else: predictions *= self.labels_std['output'] predictions += self.labels_mean['output'] if self.task == 'regression': # Predictive variance pred_var = predictions_var + mc_dropout_uncertainty # epistemic plus aleatoric uncertainty pred_uncertainty = np.sqrt(pred_var) # Convert back to std error # final correction from variance to standard derivation mc_dropout_uncertainty = np.sqrt(mc_dropout_uncertainty) predictive_uncertainty = np.sqrt(predictions_var) elif self.task == 'classification': # we want entropy for classification uncertainty predicted_class = np.argmax(predictions, axis=1) mc_dropout_uncertainty = np.ones_like(predicted_class, dtype=float) predictive_uncertainty = np.ones_like(predicted_class, dtype=float) # center variance predictions_var -= 1. for i in range(predicted_class.shape[0]): all_prediction = np.array(predictions[i, :]) mc_dropout_uncertainty[i] = -np.sum( all_prediction * np.log(all_prediction)) predictive_uncertainty[i] = predictions_var[i, predicted_class[i]] pred_uncertainty = mc_dropout_uncertainty + predictive_uncertainty # We only want the predicted class back predictions = predicted_class elif self.task == 'binary_classification': # we want entropy for classification uncertainty, so need prediction in logits space mc_dropout_uncertainty = -np.sum(predictions * np.log(predictions), axis=0) # need to activate before round to int so that the prediction is always 0 or 1 predictions = np.rint(sigmoid(predictions)) predictive_uncertainty = predictions_var pred_uncertainty = mc_dropout_uncertainty + predictions_var else: raise AttributeError('Unknown Task') return predictions, { 'total': pred_uncertainty, 'model': mc_dropout_uncertainty, 'predictive': predictive_uncertainty } def evaluate(self, input_data, labels, inputs_err=None, labels_err=None): """ Evaluate neural network by provided input data and labels and get back a metrics score :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :return: metrics score dictionary :rtype: dict :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) input_data = {"input": input_data} labels = {"output": labels} # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_input_err = inputs_err / self.input_std['input'] norm_labels_err = labels_err / self.labels_std['output'] norm_data.update({ "input_err": norm_input_err, "labels_err": norm_labels_err }) norm_labels.update({"variance_output": norm_labels["output"]}) total_num = input_data['input'].shape[0] eval_batchsize = self.batch_size if total_num > self.batch_size else total_num steps = total_num // self.batch_size if total_num > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") # suppress pfor warning from TF old_level = tf.get_logger().level tf.get_logger().setLevel('ERROR') evaluate_generator = BayesianCNNDataGenerator( batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels]) scores = self.keras_model.evaluate(evaluate_generator) tf.get_logger().setLevel(old_level) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = self.keras_model.metrics_names print( f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed' ) return list_to_dict(funcname, scores) @deprecated_copy_signature(fit) def train(self, *args, **kwargs): return self.fit(*args, **kwargs) @deprecated_copy_signature(fit_on_batch) def train_on_batch(self, *args, **kwargs): return self.fit_on_batch(*args, **kwargs) @deprecated_copy_signature(predict) def test(self, *args, **kwargs): return self.predict(*args, **kwargs)
class ConvVAEBase(NeuralNetMaster, ABC): """Top-level class for a Convolutional Variational Autoencoder""" def __init__(self): """ NAME: __init__ PURPOSE: To define astroNN Convolutional Variational Autoencoder HISTORY: 2018-Jan-06 - Written - Henry Leung (University of Toronto) """ super().__init__() self.name = 'Convolutional Variational Autoencoder' self._model_type = 'CVAE' self.initializer = None self.activation = None self._last_layer_activation = None self.num_filters = None self.filter_len = None self.pool_length = None self.num_hidden = None self.reduce_lr_epsilon = None self.reduce_lr_min = None self.reduce_lr_patience = None self.l2 = None self.latent_dim = None self.val_size = 0.1 self.dropout_rate = 0.0 self.keras_vae = None self.keras_encoder = None self.keras_decoder = None self.loss = None self.input_shape = None self.input_norm_mode = 255 self.labels_norm_mode = 255 self.input_mean = None self.input_std = None self.labels_mean = None self.labels_std = None def compile(self, optimizer=None, loss=None, metrics=None, loss_weights=None, sample_weight_mode=None): self.keras_model, self.keras_encoder, self.keras_decoder = self.model() if optimizer is not None: self.optimizer = optimizer elif self.optimizer is None or self.optimizer == 'adam': self.optimizer = Adam(lr=self.lr, beta_1=self.beta_1, beta_2=self.beta_2, epsilon=self.optimizer_epsilon, decay=0.0) if self.loss is None: self.loss = mean_squared_error self.keras_model.compile(loss=self.loss, optimizer=self.optimizer) return None def pre_training_checklist_child(self, input_data, input_recon_target): if self.task == 'classification': raise RuntimeError( 'astroNN VAE does not support classification task') elif self.task == 'binary_classification': raise RuntimeError( 'astroNN VAE does not support binary classification task') self.pre_training_checklist_master(input_data, input_recon_target) if isinstance(input_data, H5Loader): self.targetname = input_data.target input_data, input_recon_target = input_data.load() # check if exists (exists mean fine-tuning, so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(input_recon_target) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(input_recon_target, calc=False) if self.keras_model is None: # only compiler if there is no keras_model, e.g. fine-tuning does not required self.compile() self.train_idx, self.val_idx = train_test_split( np.arange(self.num_train), test_size=self.val_size) self.training_generator = CVAEDataGenerator(self.batch_size).generate( norm_data[self.train_idx], norm_labels[self.train_idx]) self.validation_generator = CVAEDataGenerator( self.batch_size).generate(norm_data[self.val_idx], norm_labels[self.val_idx]) return input_data, input_recon_target def train(self, input_data, input_recon_target): # Call the checklist to create astroNN folder and save parameters self.pre_training_checklist_child(input_data, input_recon_target) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, min_delta=self.reduce_lr_epsilon, patience=self.reduce_lr_patience, min_lr=self.reduce_lr_min, mode='min', verbose=2) self.virtual_cvslogger = VirutalCSVLogger() self.__callbacks = [reduce_lr, self.virtual_cvslogger ] # default must have unchangeable callbacks if self.callbacks is not None: if isinstance(self.callbacks, list): self.__callbacks.extend(self.callbacks) else: self.__callbacks.append(self.callbacks) start_time = time.time() self.keras_model.fit_generator( generator=self.training_generator, steps_per_epoch=self.num_train // self.batch_size, validation_data=self.validation_generator, validation_steps=self.val_num // self.batch_size, epochs=self.max_epochs, verbose=self.verbose, workers=os.cpu_count(), callbacks=self.__callbacks, use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training, {(time.time() - start_time):.{2}f}s in total' ) if self.autosave is True: # Call the post training checklist to save parameters self.save() return None def post_training_checklist_child(self): astronn_model = 'model_weights.h5' self.keras_model.save(self.fullfilepath + astronn_model) print(astronn_model + f' saved to {(self.fullfilepath + astronn_model)}') self.hyper_txt.write(f"Dropout Rate: {self.dropout_rate} \n") self.hyper_txt.flush() self.hyper_txt.close() data = { 'id': self.__class__.__name__, 'pool_length': self.pool_length, 'filterlen': self.filter_len, 'filternum': self.num_filters, 'hidden': self.num_hidden, 'input': self.input_shape, 'labels': self.labels_shape, 'task': self.task, 'input_mean': self.input_mean.tolist(), 'labels_mean': self.labels_mean.tolist(), 'input_std': self.input_std.tolist(), 'labels_std': self.labels_std.tolist(), 'valsize': self.val_size, 'targetname': self.targetname, 'dropout_rate': self.dropout_rate, 'l2': self.l2, 'input_norm_mode': self.input_norm_mode, 'labels_norm_mode': self.labels_norm_mode, 'batch_size': self.batch_size, 'latent': self.latent_dim } with open(self.fullfilepath + '/astroNN_model_parameter.json', 'w') as f: json.dump(data, f, indent=4, sort_keys=True) def test(self, input_data): self.pre_testing_checklist_master() input_data = np.atleast_2d(input_data) if self.input_normalizer is not None: input_array = self.input_normalizer.normalize(input_data, calc=False) else: # Prevent shallow copy issue input_array = np.array(input_data) input_array -= self.input_mean input_array /= self.input_std total_test_num = input_data.shape[0] # Number of testing data # for number of training data smaller than batch_size if input_data.shape[0] < self.batch_size: self.batch_size = input_data.shape[0] # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // self.batch_size) * self.batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator predictions = np.zeros((total_test_num, self.labels_shape, 1)) # Data Generator for prediction prediction_generator = CVAEPredDataGenerator(self.batch_size).generate( input_array[:data_gen_shape]) predictions[:data_gen_shape] = np.asarray( self.keras_model.predict_generator(prediction_generator, steps=input_array.shape[0] // self.batch_size)) if remainder_shape != 0: remainder_data = input_array[data_gen_shape:] # assume its caused by mono images, so need to expand dim by 1 if len(input_array[0].shape) != len(self.input_shape): remainder_data = np.expand_dims(remainder_data, axis=-1) result = self.keras_model.predict(remainder_data) predictions[data_gen_shape:] = result if self.labels_normalizer is not None: predictions[:, :, 0] = self.labels_normalizer.denormalize( predictions[:, :, 0]) else: predictions[:, :, 0] *= self.labels_std predictions[:, :, 0] += self.labels_mean return predictions def test_encoder(self, input_data): self.pre_testing_checklist_master() # Prevent shallow copy issue if self.input_normalizer is not None: input_array = self.input_normalizer.normalize(input_data, calc=False) else: # Prevent shallow copy issue input_array = np.array(input_data) input_array -= self.input_mean input_array /= self.input_std total_test_num = input_data.shape[0] # Number of testing data # for number of training data smaller than batch_size if input_data.shape[0] < self.batch_size: self.batch_size = input_data.shape[0] # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // self.batch_size) * self.batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator encoding = np.zeros((total_test_num, self.latent_dim)) # Data Generator for prediction prediction_generator = CVAEPredDataGenerator(self.batch_size).generate( input_array[:data_gen_shape]) encoding[:data_gen_shape] = np.asarray( self.keras_encoder.predict_generator(prediction_generator, steps=input_array.shape[0] // self.batch_size)) if remainder_shape != 0: remainder_data = input_array[data_gen_shape:] # assume its caused by mono images, so need to expand dim by 1 if len(input_array[0].shape) != len(self.input_shape): remainder_data = np.expand_dims(remainder_data, axis=-1) result = self.keras_encoder.predict(remainder_data) encoding[data_gen_shape:] = result return encoding
def fit_on_batch(self, input_data, labels, inputs_err=None, labels_err=None, sample_weights=None): """ Train a Bayesian neural network by running a single gradient update on all of your data, suitable for fine-tuning :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param inputs_err: Error for input_data (if any), same shape with input_data. :type inputs_err: Union([NoneType, ndarray]) :param labels_err: Labels error (if any) :type labels_err: Union([NoneType, ndarray]) :param sample_weights: Sample weights (if any) :type sample_weights: Union([NoneType, ndarray]) :return: None :rtype: NoneType :History: | 2018-Aug-25 - Written - Henry Leung (University of Toronto) """ self.has_model_check() if inputs_err is None: inputs_err = np.zeros_like(input_data) if labels_err is None: labels_err = np.zeros_like(labels) input_data = { "input": input_data, "input_err": inputs_err, "labels_err": labels_err } labels = {"output": labels, "variance_output": labels} # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) # No need to care about Magic number as loss function looks for magic num in y_true only norm_data.update({ "input_err": (input_data['input_err'] / self.input_std['input']), "labels_err": input_data['labels_err'] / self.labels_std['output'] }) norm_labels.update({"variance_output": norm_labels['output']}) norm_data = self._tensor_dict_sanitize(norm_data, self.keras_model.input_names) norm_labels = self._tensor_dict_sanitize(norm_labels, self.keras_model.output_names) start_time = time.time() fit_generator = BayesianCNNDataGenerator( batch_size=input_data['input'].shape[0], shuffle=False, steps_per_epoch=1, data=[norm_data, norm_labels], sample_weights=sample_weights) score = self.keras_model.fit(fit_generator, epochs=1, verbose=self.verbose, workers=os.cpu_count(), use_multiprocessing=MULTIPROCESS_FLAG) print( f'Completed Training on Batch, {(time.time() - start_time):.{2}f}s in total' ) return None
def pre_training_checklist_child(self, input_data, labels, sample_weights): # on top of checklist, convert input_data/labels to dict input_data, labels = self.pre_training_checklist_master(input_data, labels) # check if exists (existing means the model has already been trained (e.g. fine-tuning) # so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) if self.keras_model is None: # only compile if there is no keras_model, e.g. fine-tuning does not required self.compile() norm_data = self._tensor_dict_sanitize(norm_data, self.keras_model.input_names) norm_labels = self._tensor_dict_sanitize(norm_labels, self.keras_model.output_names) self.train_idx, self.val_idx = train_test_split(np.arange(self.num_train + self.val_num), test_size=self.val_size) norm_data_training = {} norm_data_val = {} norm_labels_training = {} norm_labels_val = {} for name in norm_data.keys(): norm_data_training.update({name: norm_data[name][self.train_idx]}) norm_data_val.update({name: norm_data[name][self.val_idx]}) for name in norm_labels.keys(): norm_labels_training.update({name: norm_labels[name][self.train_idx]}) norm_labels_val.update({name: norm_labels[name][self.val_idx]}) if sample_weights is not None: sample_weights_training = sample_weights[self.train_idx] sample_weights_val = sample_weights[self.val_idx] else: sample_weights_training = None sample_weights_val = None self.training_generator = CNNDataGenerator( batch_size=self.batch_size, shuffle=True, steps_per_epoch=self.num_train // self.batch_size, data=[norm_data_training, norm_labels_training], manual_reset=False, sample_weights=sample_weights_training) val_batchsize = self.batch_size if len(self.val_idx) > self.batch_size else len(self.val_idx) self.validation_generator = CNNDataGenerator( batch_size=val_batchsize, shuffle=False, steps_per_epoch=max(self.val_num // self.batch_size, 1), data=[norm_data_val, norm_labels_val], manual_reset=True, sample_weights=sample_weights_val) return input_data, labels
class CNNBase(NeuralNetMaster, ABC): """Top-level class for a convolutional neural network""" def __init__(self): """ NAME: __init__ PURPOSE: To define astroNN convolutional neural network HISTORY: 2018-Jan-06 - Written - Henry Leung (University of Toronto) """ super().__init__() self.name = 'Convolutional Neural Network' self._model_type = 'CNN' self._model_identifier = None self.initializer = None self.activation = None self._last_layer_activation = None self.num_filters = None self.filter_len = None self.pool_length = None self.num_hidden = None self.reduce_lr_epsilon = None self.reduce_lr_min = None self.reduce_lr_patience = None self.l1 = None self.l2 = None self.maxnorm = None self.dropout_rate = 0.0 self.val_size = 0.1 self.early_stopping_min_delta = 0.0001 self.early_stopping_patience = 4 self.input_norm_mode = 1 self.labels_norm_mode = 2 def compile(self, optimizer=None, loss=None, metrics=None, weighted_metrics=None, loss_weights=None, sample_weight_mode=None): if optimizer is not None: self.optimizer = optimizer elif self.optimizer is None or self.optimizer == 'adam': self.optimizer = Adam(learning_rate=self.lr, beta_1=self.beta_1, beta_2=self.beta_2, epsilon=self.optimizer_epsilon, decay=0.0) if metrics is not None: self.metrics = metrics if self.task == 'regression': self._last_layer_activation = 'linear' loss_func = mean_squared_error if not loss else loss self.metrics = [mean_absolute_error, mean_error] if not self.metrics else self.metrics elif self.task == 'classification': self._last_layer_activation = 'softmax' loss_func = categorical_crossentropy if not loss else loss self.metrics = [categorical_accuracy] if not self.metrics else self.metrics elif self.task == 'binary_classification': self._last_layer_activation = 'sigmoid' loss_func = binary_crossentropy if not loss else loss self.metrics = [binary_accuracy] if not self.metrics else self.metrics else: raise RuntimeError('Only "regression", "classification" and "binary_classification" are supported') self.keras_model = self.model() self.keras_model.compile(loss=loss_func, optimizer=self.optimizer, metrics=self.metrics, weighted_metrics=weighted_metrics, loss_weights=loss_weights, sample_weight_mode=sample_weight_mode) # inject custom training step if needed try: self.custom_train_step() except NotImplementedError: pass except TypeError: self.keras_model.train_step = self.custom_train_step return None def pre_training_checklist_child(self, input_data, labels, sample_weights): # on top of checklist, convert input_data/labels to dict input_data, labels = self.pre_training_checklist_master(input_data, labels) # check if exists (existing means the model has already been trained (e.g. fine-tuning) # so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) if self.keras_model is None: # only compile if there is no keras_model, e.g. fine-tuning does not required self.compile() norm_data = self._tensor_dict_sanitize(norm_data, self.keras_model.input_names) norm_labels = self._tensor_dict_sanitize(norm_labels, self.keras_model.output_names) self.train_idx, self.val_idx = train_test_split(np.arange(self.num_train + self.val_num), test_size=self.val_size) norm_data_training = {} norm_data_val = {} norm_labels_training = {} norm_labels_val = {} for name in norm_data.keys(): norm_data_training.update({name: norm_data[name][self.train_idx]}) norm_data_val.update({name: norm_data[name][self.val_idx]}) for name in norm_labels.keys(): norm_labels_training.update({name: norm_labels[name][self.train_idx]}) norm_labels_val.update({name: norm_labels[name][self.val_idx]}) if sample_weights is not None: sample_weights_training = sample_weights[self.train_idx] sample_weights_val = sample_weights[self.val_idx] else: sample_weights_training = None sample_weights_val = None self.training_generator = CNNDataGenerator( batch_size=self.batch_size, shuffle=True, steps_per_epoch=self.num_train // self.batch_size, data=[norm_data_training, norm_labels_training], manual_reset=False, sample_weights=sample_weights_training) val_batchsize = self.batch_size if len(self.val_idx) > self.batch_size else len(self.val_idx) self.validation_generator = CNNDataGenerator( batch_size=val_batchsize, shuffle=False, steps_per_epoch=max(self.val_num // self.batch_size, 1), data=[norm_data_val, norm_labels_val], manual_reset=True, sample_weights=sample_weights_val) return input_data, labels def fit(self, input_data, labels, sample_weights=None): """ Train a Convolutional neural network :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param sample_weights: Sample weights (if any) :type sample_weights: Union([NoneType, ndarray]) :return: None :rtype: NoneType :History: 2017-Dec-06 - Written - Henry Leung (University of Toronto) """ # Call the checklist to create astroNN folder and save parameters self.pre_training_checklist_child(input_data, labels, sample_weights) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, min_delta=self.reduce_lr_epsilon, patience=self.reduce_lr_patience, min_lr=self.reduce_lr_min, mode='min', verbose=2) early_stopping = EarlyStopping(monitor='val_loss', min_delta=self.early_stopping_min_delta, patience=self.early_stopping_patience, verbose=2, mode='min') self.virtual_cvslogger = VirutalCSVLogger() self.__callbacks = [reduce_lr, self.virtual_cvslogger] # default must have unchangeable callbacks if self.callbacks is not None: if isinstance(self.callbacks, list): self.__callbacks.extend(self.callbacks) else: self.__callbacks.append(self.callbacks) start_time = time.time() self.history = self.keras_model.fit(x=self.training_generator, validation_data=self.validation_generator, epochs=self.max_epochs, verbose=self.verbose, workers=os.cpu_count(), callbacks=self.__callbacks, use_multiprocessing=MULTIPROCESS_FLAG) print(f'Completed Training, {(time.time() - start_time):.{2}f}s in total') if self.autosave is True: # Call the post training checklist to save parameters self.save() return None def fit_on_batch(self, input_data, labels, sample_weights=None): """ Train a neural network by running a single gradient update on all of your data, suitable for fine-tuning :param input_data: Data to be trained with neural network :type input_data: ndarray :param labels: Labels to be trained with neural network :type labels: ndarray :param sample_weights: Sample weights (if any) :type sample_weights: Union([NoneType, ndarray]) :return: None :rtype: NoneType :History: 2018-Aug-22 - Written - Henry Leung (University of Toronto) """ input_data, labels = self.pre_training_checklist_master(input_data, labels) # check if exists (existing means the model has already been trained (e.g. fine-tuning), # so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) start_time = time.time() fit_generator = CNNDataGenerator(batch_size=input_data['input'].shape[0], shuffle=False, steps_per_epoch=1, data=[norm_data, norm_labels], sample_weights=sample_weights) scores = self.keras_model.fit(x=fit_generator, epochs=1, verbose=self.verbose, workers=os.cpu_count(), use_multiprocessing=MULTIPROCESS_FLAG) print(f'Completed Training on Batch, {(time.time() - start_time):.{2}f}s in total') return None def post_training_checklist_child(self): self.keras_model.save(self.fullfilepath + _astroNN_MODEL_NAME) print(_astroNN_MODEL_NAME + f' saved to {(self.fullfilepath + _astroNN_MODEL_NAME)}') self.hyper_txt.write(f"Dropout Rate: {self.dropout_rate} \n") self.hyper_txt.flush() self.hyper_txt.close() data = {'id': self.__class__.__name__ if self._model_identifier is None else self._model_identifier, 'pool_length': self.pool_length, 'filterlen': self.filter_len, 'filternum': self.num_filters, 'hidden': self.num_hidden, 'input': self._input_shape, 'labels': self._labels_shape, 'task': self.task, 'last_layer_activation': self._last_layer_activation, 'activation': self.activation, 'input_mean': dict_np_to_dict_list(self.input_mean), 'labels_mean': dict_np_to_dict_list(self.labels_mean), 'input_std': dict_np_to_dict_list(self.input_std), 'labels_std': dict_np_to_dict_list(self.labels_std), 'valsize': self.val_size, 'targetname': self.targetname, 'dropout_rate': self.dropout_rate, 'l1': self.l1, 'l2': self.l2, 'maxnorm': self.maxnorm, 'input_norm_mode': self.input_normalizer.normalization_mode, 'labels_norm_mode': self.labels_normalizer.normalization_mode, 'input_names': self.input_names, 'output_names': self.output_names, 'batch_size': self.batch_size} with open(self.fullfilepath + '/astroNN_model_parameter.json', 'w') as f: json.dump(data, f, indent=4, sort_keys=True) def predict(self, input_data): """ Use the neural network to do inference :param input_data: Data to be inferred with neural network :type input_data: ndarray :return: prediction and prediction uncertainty :rtype: ndarry :History: 2017-Dec-06 - Written - Henry Leung (University of Toronto) """ self.has_model_check() input_data = self.pre_testing_checklist_master(input_data) input_array = self.input_normalizer.normalize(input_data, calc=False) total_test_num = input_data['input'].shape[0] # Number of testing data # for number of training data smaller than batch_size if total_test_num < self.batch_size: self.batch_size = total_test_num # Due to the nature of how generator works, no overlapped prediction data_gen_shape = (total_test_num // self.batch_size) * self.batch_size remainder_shape = total_test_num - data_gen_shape # Remainder from generator # TODO: named output???? predictions = np.zeros((total_test_num, self._labels_shape['output'])) norm_data_main = {} norm_data_remainder = {} for name in input_array.keys(): norm_data_main.update({name: input_array[name][:data_gen_shape]}) norm_data_remainder.update({name: input_array[name][data_gen_shape:]}) norm_data_main = self._tensor_dict_sanitize(norm_data_main, self.keras_model.input_names) norm_data_remainder = self._tensor_dict_sanitize(norm_data_remainder, self.keras_model.input_names) # Data Generator for prediction with tqdm(total=total_test_num, unit="sample") as pbar: prediction_generator = CNNPredDataGenerator(batch_size=self.batch_size, shuffle=False, steps_per_epoch=total_test_num // self.batch_size, data=[norm_data_main], pbar=pbar) predictions[:data_gen_shape] = np.asarray(self.keras_model.predict(prediction_generator)) if remainder_shape != 0: remainder_generator = CNNPredDataGenerator(batch_size=remainder_shape, shuffle=False, steps_per_epoch=1, data=[norm_data_remainder], pbar=pbar) predictions[data_gen_shape:] = np.asarray(self.keras_model.predict(remainder_generator)) if self.labels_normalizer is not None: predictions = self.labels_normalizer.denormalize(list_to_dict(self.keras_model.output_names, predictions)) else: predictions *= self.labels_std predictions += self.labels_mean return predictions['output'] def evaluate(self, input_data, labels): """ Evaluate neural network by provided input data and labels and get back a metrics score :param input_data: Data to be inferred with neural network :type input_data: ndarray :param labels: labels :type labels: ndarray :return: metrics score dictionary :rtype: dict :History: 2018-May-20 - Written - Henry Leung (University of Toronto) """ self.has_model_check() input_data = list_to_dict(self.keras_model.input_names, input_data) labels = list_to_dict(self.keras_model.output_names, labels) # check if exists (existing means the model has already been trained (e.g. fine-tuning), so we do not need calculate mean/std again) if self.input_normalizer is None: self.input_normalizer = Normalizer(mode=self.input_norm_mode) self.labels_normalizer = Normalizer(mode=self.labels_norm_mode) norm_data = self.input_normalizer.normalize(input_data) self.input_mean, self.input_std = self.input_normalizer.mean_labels, self.input_normalizer.std_labels norm_labels = self.labels_normalizer.normalize(labels) self.labels_mean, self.labels_std = self.labels_normalizer.mean_labels, self.labels_normalizer.std_labels else: norm_data = self.input_normalizer.normalize(input_data, calc=False) norm_labels = self.labels_normalizer.normalize(labels, calc=False) norm_data = self._tensor_dict_sanitize(norm_data, self.keras_model.input_names) norm_labels = self._tensor_dict_sanitize(norm_labels, self.keras_model.output_names) total_num = input_data['input'].shape[0] eval_batchsize = self.batch_size if total_num > self.batch_size else total_num steps = total_num // self.batch_size if total_num > self.batch_size else 1 start_time = time.time() print("Starting Evaluation") evaluate_generator = CNNDataGenerator(batch_size=eval_batchsize, shuffle=False, steps_per_epoch=steps, data=[norm_data, norm_labels]) scores = self.keras_model.evaluate(evaluate_generator) if isinstance(scores, float): # make sure scores is iterable scores = list(str(scores)) outputname = self.keras_model.output_names funcname = self.keras_model.metrics_names print(f'Completed Evaluation, {(time.time() - start_time):.{2}f}s elapsed') return list_to_dict(funcname, scores) @deprecated_copy_signature(fit) def train(self, *args, **kwargs): return self.fit(*args, **kwargs) @deprecated_copy_signature(fit_on_batch) def train_on_batch(self, *args, **kwargs): return self.fit_on_batch(*args, **kwargs) @deprecated_copy_signature(predict) def test(self, *args, **kwargs): return self.predict(*args, **kwargs)