def compile_models(self, verbose: int = 0) -> NoReturn: """Compiles the neural network architectures. Arguments ---------- verbose: Verbosity level. """ print("\nCompile the following NOMU_DJ Models:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) self.compile_mean_models() self.compile_sigma_models() # update all parameters of main architectures self.update_all_weights() print()
def fit_models( self, x: np.array, y: np.array, verbose: int = 0, ) -> NoReturn: """Fits the neural network architectures to specified data. Arguments ---------- x : input data (features) y : output data (targets). verbose : Level of verbosity. """ print("\nFit the following MC Dropout Models:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key in self.models.keys(): print(key) p = self.parameters[key] model = self.models[key] # fit model start = datetime.now() history = model.fit( x, y, epochs=p["epochs"], batch_size=p["batch_size"], verbose=verbose ) end = datetime.now() diff = end - start print( "Elapsed: {}d {}h:{}m:{}s".format(*timediff_d_h_m_s(diff)), "(" + datetime.now().strftime("%H:%M %d-%m-%Y") + ")", ) self.models[key] = model if self.histories[key] is None: self.histories[key] = history.history else: self.histories[key] = { loss_key: loss_value + history.history[loss_key] for loss_key, loss_value in self.histories[key].items() } self.parameters[key]["actual_epochs"] = len(self.histories[key]["loss"]) print()
def compile_models(self, verbose: int = 0) -> NoReturn: """Compiles the neural network architectures. Arguments ---------- verbose: Verbosity level. """ print("\nCompile the following MC Dropout Models:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key in self.models.keys(): print(key) p = self.parameters[key] model = self.models[key] # compile model # tensorflow version 2.1 # (i) use adam if p["optimizer"] == "Adam": optimizer = Adam( learning_rate=p["learning_rate"], beta_1=p["beta_1"], beta_2=p["beta_2"], epsilon=p["epsilon"], amsgrad=p["amsgrad"], name=p["optimizer"], clipnorm=p["clipnorm"], ) if p["optimizer"] == "SGD": # (ii) plain vanilla gd optimizer = SGD( learning_rate=p["learning_rate"], momentum=p["momentum"], nesterov=p["nesterov"], name=p["optimizer"], clipnorm=p["clipnorm"], ) model.compile( optimizer=optimizer, loss=p["loss"], experimental_run_tf_function=False ) self.models[key] = model print()
def fit_models( self, x: np.array, y: np.array, verbose: int = 0, x_min_aug: float = -1 - 0.1, x_max_aug: float = 1 + 0.1, ) -> NoReturn: """Fits the neural network architectures to specified data. Arguments ---------- x : input data (features) y : output data (targets). verbose : Level of verbosity. x_min_aug : Min of box bound for augmented data points. x_max_aug : Max of box bound for augmented data points. """ # fit NOMU_dj models print("\nFit the following NOMU_DJ Models:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) # fit mean models self.fit_mean_models(x=x, y=y, verbose=verbose) # set parameters of main architecture in sigma model to the ones of the mean model self.update_sigma_weights() # fit sigma model self.fit_sigma_models(x=x, y=y, verbose=verbose, x_min_aug=x_min_aug, x_max_aug=x_max_aug) # update all parameters of main architectures self.update_all_weights() print()
def initialize_models(self, verbose: int = 0) -> NoReturn: """Initializes the kernels for the GPs. Arguments ---------- verbose : Verbosity level. """ print("\nInitialize the following Gaussian Processes:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key in self.models.keys(): print(key) p = self.parameters[key] if p["kernel"] == "rbf": if p["whitekernel"]: K = C( constant_value=p["constant_value"], constant_value_bounds=p["constant_value_bounds"], ) * RBF( length_scale=p["length_scale"], length_scale_bounds=p["length_scale_bounds"], ) + WhiteKernel( noise_level=p["noise_level"], noise_level_bounds=p["noise_level_bounds"], ) else: K = C( constant_value=p["constant_value"], constant_value_bounds=p["constant_value_bounds"], ) * RBF( length_scale=p["length_scale"], length_scale_bounds=p["length_scale_bounds"], ) else: raise NotImplementedError("Kernel {} not implemented".format( p["kernel"])) self.initial_kernels[key] = K print()
def fit_models(self, x: np.array, y: np.array, verbose: int = 0) -> NoReturn: """Fits the GPs to specified data. Arguments ---------- x : input data (features) y : output data (targets). verbose : Level of verbosity. """ print("\nFit the following Gaussian Processes:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key, model in self.models.items(): print(key) start = datetime.now() self.models[key].fit(x, y) end = datetime.now() diff = end - start print( "Elapsed: {}d {}h:{}m:{}s".format(*timediff_d_h_m_s(diff)), "(" + datetime.now().strftime("%H:%M %d-%m-%Y") + ")", ) print()
def compile_models(self, verbose: int = 0) -> NoReturn: """Compiles the GPs. Arguments ---------- verbose: Verbosity level. """ print("\nCompile the following Gaussian Processes:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key, kernel in self.initial_kernels.items(): print(key) p = self.parameters[key] gp = GaussianProcessRegressor( kernel=kernel, alpha=p["alpha"], optimizer=p["optimizer"], n_restarts_optimizer=p["n_restarts_optimizer"], normalize_y=p["normalize_y"], copy_X_train=p["copy_X_train"], random_state=p["random_state"], ) self.models[key] = gp print()
def initialize_models( self, s: float = 0.05, activation: str = "relu", verbose: int = 0 ) -> NoReturn: """Initializes the neural network architectures. Arguments ---------- s : Interval parameters for uniform random weight initialization in the intervall [-s,s] activation : Tf activation function. verbose : Verbosity level. """ print("\nInitialize the following MC Dropout Models:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key in self.models.keys(): print(key) p = self.parameters[key] layers = p["layers"] l2reg = p["l2reg"] seed = p["seed_init"] dropout = p["dropout_prob"] # input layer x_input = Input(shape=(layers[0],), name="input_layer") # first hidden layer l = Dense( layers[1], activation=activation, name="hidden_layer_{}".format(1), kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=seed), bias_initializer=RandomUniform( minval=-s, maxval=s, seed=update_seed(seed, 1) ), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(x_input) if dropout != 0: l = Dropout(dropout)(l, training=True) # hidden layers for i, n in enumerate(layers[2:-1]): l = Dense( n, activation=activation, name="hidden_layer_{}".format(i + 2), kernel_initializer=RandomUniform( minval=-s, maxval=s, seed=update_seed(seed, 2 * i + 2) ), bias_initializer=RandomUniform( minval=-s, maxval=s, seed=update_seed(seed, 2 * i + 3) ), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(l) if dropout != 0: l = Dropout(dropout)(l, training=True) # output layer x_output = Dense( layers[-1], activation="linear", name="output_layer", kernel_initializer=RandomUniform( minval=-s, maxval=s, seed=update_seed(seed, -1) ), bias_initializer=RandomUniform( minval=-s, maxval=s, seed=update_seed(seed, -2) ), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(l) self.models[key] = Model(inputs=[x_input], outputs=x_output) if verbose > 0: print("Architecture\n") print(self.models[key].summary()) print()
def fit_models( self, x: np.array, y: np.array, verbose: int = 0, x_min_aug: float = -1 - 0.1, x_max_aug: float = 1 + 0.1, ) -> NoReturn: """Fits the neural network architectures to specified data. Arguments ---------- x : input data (features) y : output data (targets). verbose : Level of verbosity. x_min_aug : Min of box bound for augmented data points. x_max_aug : Max of box bound for augmented data points. """ # fit NOMU models print("\nFit the following NOMU Models:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key in self.models.keys(): print(key) p = self.parameters[key] model = self.models[key] # set up generator generator = DataGenerator( batch_size=p["batch_size"], x=x, y=y, n_train=p["n_train"], n_aug=p["n_aug"], MCaug=p["MCaug"], x_min_aug=x_min_aug, x_max_aug=x_max_aug, joint_loss=True, ) # fit model best_weights_callback = ReturnBestWeights(monitor="loss", verbose=1, mode="min", baseline=None) start = datetime.now() history = model.fit( x=generator, epochs=p["epochs"], verbose=verbose, callbacks=[ best_weights_callback, PredictionHistory(generator) ], ) end = datetime.now() diff = end - start print( "Elapsed: {}d {}h:{}m:{}s".format(*timediff_d_h_m_s(diff)), "(" + datetime.now().strftime("%H:%M %d-%m-%Y") + ")", ) self.models[key] = model if self.histories[key] is None: self.histories[key] = history.history else: self.histories[key] = { loss_key: loss_value + history.history[loss_key] for loss_key, loss_value in self.histories[key].items() } self.parameters[key]["actual_epochs"] = len( self.histories[key]["loss"]) print()
def compile_models(self, verbose: int = 0) -> NoReturn: """Compiles the neural network architectures. Arguments ---------- verbose: Verbosity level. """ print("\nCompile the following NOMU Models:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key in self.models.keys(): print(key) p = self.parameters[key] model = self.models[key] flag = self.flags[key] # define loss custom_loss = [ squared_loss_wrapper(flag=flag, n_train=p["n_train"], n_aug=p["n_aug"]), r_loss_wrapper( flag=flag, mu_sqr=p["mu_sqr"], mu_exp=p["mu_exp"], c_exp=p["c_exp"], n_train=p["n_train"], n_aug=p["n_aug"], stable_aug_loss=p["stable_aug_loss"], c_2=p["c_sqr_stable_aug_loss"], ), ] # compile model # tensorflow version 2.1 # (i) use adam if p["optimizer"] == "Adam": optimizer = Adam( learning_rate=p["learning_rate"], beta_1=p["beta_1"], beta_2=p["beta_2"], epsilon=p["epsilon"], amsgrad=p["amsgrad"], name=p["optimizer"], clipnorm=p["clipnorm"], ) if p["optimizer"] == "SGD": # (ii) plain vanilla gd optimizer = SGD( learning_rate=p["learning_rate"], momentum=p["momentum"], nesterov=p["nesterov"], name=p["optimizer"], clipnorm=p["clipnorm"], ) model.compile( optimizer=optimizer, loss=custom_loss, experimental_run_tf_function=False, ) self.models[key] = model print()
def initialize_models(self, s: float = 0.05, activation: str = "relu", verbose: int = 0) -> NoReturn: """Initializes the neural network architectures. Arguments ---------- s : Interval parameters for uniform random weight initialization in the intervall [-s,s] activation : Tf activation function. verbose : Verbosity level. """ print("\nInitialize the following NOMU Models:") print( "**************************************************************************" ) if verbose > 0: for key in self.models.keys(): print(key) pretty_print_dict(self.parameters[key]) print() print( "**************************************************************************" ) for key in self.models.keys(): print(key) p = self.parameters[key] layers = p["layers"] side_layers = p["side_layers"] l2reg = p["l2reg"] l2reg_sig = p["l2reg_sig"] seed = p["seed_init"] RSN = p["RSN"] dropout = p["dropout_prob"] # input layer x_input = Input(shape=(layers[0], ), name="input_layer") flag = Input(shape=(1, ), name="flag_input") # main architecture ------------------------------------------------------- # first hidden layer y = Dense( layers[1], activation=activation, name="hidden_layer_{}".format(1), kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=seed), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed(seed, 1)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), trainable=not (RSN), )(x_input) if dropout != 0 and dropout is not None: y = Dropout(dropout)(y, training=False) # hidden layers for i, n in enumerate(layers[2:-1]): y = Dense( n, activation=activation, name="hidden_layer_{}".format(i + 2), kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * i + 2)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * i + 3)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), trainable=not (RSN), )(y) if dropout != 0 and dropout is not None: y = Dropout(dropout)(y, training=False) # output layer y_output = Dense( layers[-1], activation="linear", name="output_layer", kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed(seed, -1)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed(seed, -2)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(y) # side architecture r ----------------------------------------------------- # set new seed seed = update_seed(seed, 2**6) # first hidden layer r = Dense( side_layers[1], activation=activation, name="r_hidden_layer_{}".format(1), kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=seed), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed(seed, 1)), kernel_regularizer=l2(l2reg_sig), bias_regularizer=l2(l2reg_sig), trainable=not (RSN), )(x_input) # hidden layers for i, n in enumerate(side_layers[2:-1]): r = Dense( n, activation=activation, name="r_hidden_layer_{}".format(i + 2), kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * i + 2)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * i + 3)), kernel_regularizer=l2(l2reg_sig), bias_regularizer=l2(l2reg_sig), trainable=not (RSN), )(r) # concatenate last hidden of y and r y_r_concat = concatenate([y, r]) r_output = Dense( side_layers[-1], activation="linear", name="r_output_layer", kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed(seed, -1)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed(seed, -2)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(y_r_concat) self.models[key] = Model(inputs=[x_input, flag], outputs=[y_output, r_output]) self.flags[key] = flag print()
def fit_models( self, x: np.array, y: np.array, verbose: int = 0, ) -> NoReturn: """Fits the neural network architectures to specified data. Arguments ---------- x : input data (features) y : output data (targets). verbose : Level of verbosity. """ print("\nFit the following Deep Ensembles:") print( "**************************************************************************" ) if verbose > 0: for ensemble_key in self.models.keys(): print(ensemble_key) pretty_print_dict(self.parameters[ensemble_key]) print() print( "**************************************************************************" ) for ensemble_key, ensemble in self.models.items(): print(ensemble_key) p = self.parameters[ensemble_key] start = datetime.now() history = OrderedDict() for model_key, model in ensemble.items(): print("Fit {}".format(model_key)) tmp = model.fit( x, y, epochs=p["epochs"], batch_size=p["batch_size"], verbose=verbose, callbacks=[PredictionHistory_DE(x, y)], ) history[model_key] = tmp.history end = datetime.now() diff = end - start print( "Elapsed: {}d {}h:{}m:{}s".format(*timediff_d_h_m_s(diff)), "(" + datetime.now().strftime("%H:%M %d-%m-%Y") + ")", ) self.models[ensemble_key] = ensemble if self.histories[ensemble_key] is None: self.histories[ensemble_key] = history else: self.histories[ensemble_key] = { model_key: { loss_key: loss_value + history[model_key][loss_key] for loss_key, loss_value in self.histories[ensemble_key][model_key].items() } for model_key in self.models[ensemble_key].keys() } self.parameters[ensemble_key]["actual_epochs"] = len( self.histories[ensemble_key][list( self.histories[ensemble_key].keys())[0]]["loss"]) print() print()
def compile_models(self, verbose: int = 0) -> NoReturn: """Compiles the neural network architectures. Arguments ---------- verbose: Verbosity level. """ print("\nCompile the following Deep Ensembles:") print( "**************************************************************************" ) if verbose > 0: for ensemble_key in self.models.keys(): print(ensemble_key) pretty_print_dict(self.parameters[ensemble_key]) print() print( "**************************************************************************" ) for ensemble_key in self.models.keys(): print(ensemble_key) p = self.parameters[ensemble_key] for model_key, model in self.models[ensemble_key].items(): print("Compile:", model_key) if p["loss"] == "nll": loss_DE = gaussian_nll elif p["loss"] == "mse": loss_DE = "mse" else: raise NotImplementedError( "{} loss is not implemented.".format(p["loss"])) # tensorflow version 2.1 if p["optimizer"] == "Adam": optimizer = Adam( learning_rate=p["learning_rate"], beta_1=p["beta_1"], beta_2=p["beta_2"], epsilon=p["epsilon"], amsgrad=p["amsgrad"], name=p["optimizer"], clipnorm=p["clipnorm"], ) elif p["optimizer"] == "SGD": optimizer = SGD( learning_rate=p["learning_rate"], momentum=p["momentum"], nesterov=p["nesterov"], name=p["optimizer"], clipnorm=p["clipnorm"], ) else: raise NotImplementedError( "{} optimizer is not implemented.".format( p["optimizer"])) model.compile( optimizer=optimizer, loss=loss_DE, experimental_run_tf_function=False, ) print()
def initialize_models(self, s: float = 0.05, activation: str = "relu", verbose: int = 0) -> NoReturn: """Initializes the neural network architectures. Arguments ---------- s : Interval parameters for uniform random weight initialization in the intervall [-s,s] activation : Tf activation function. verbose : Verbosity level. """ print("\nInitialize the following Deep Ensembles:") print( "**************************************************************************" ) if verbose > 0: for ensemble_key in self.models.keys(): print(ensemble_key) pretty_print_dict(self.parameters[ensemble_key]) print() print( "**************************************************************************" ) for ensemble_key in self.models.keys(): print(ensemble_key) p = self.parameters[ensemble_key] layers = p["layers"] l2reg = p["l2reg"] number_of_networks = p["number_of_networks"] loss = p["loss"] softplus_min_var = p["softplus_min_var"] l2reg = p["l2reg"] seed = p["seed_init"] ensemble = {} for i in range(1, number_of_networks + 1): modelname = "NN_{}".format(i) print("Initialize:", modelname) # input layer x_input = Input(shape=(layers[0], ), name=modelname + "_input_layer") # first hidden layer l = Dense( layers[1], activation=activation, name=modelname + "_hidden_layer_{}".format(1), kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed(seed, 0)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed(seed, 1)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(x_input) # hidden layers for i, n in enumerate(layers[2:-1]): l = Dense( n, activation=activation, name=modelname + "_hidden_layer_{}".format(i + 2), kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * i + 2)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * i + 3)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(l) if loss == "nll": # output layer with two parameters for each output dimension in case loss==nll: mu_output = Dense( layers[-1], activation="linear", name=modelname + "_output_layer_mu", kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * (i + 1) + 2)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * (i + 1) + 3)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(l) sigma_output = Dense( layers[-1], activation=softplus_wrapper(min_var=softplus_min_var), name=modelname + "_output_layer_sigma", kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * (i + 2) + 2)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * (i + 2) + 3)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(l) x_output = concatenate([mu_output, sigma_output]) elif loss == "mse": # output layer with two parameters for each output dimension in case loss==nll: mu_output = Dense( layers[-1], activation="linear", name=modelname + "_output_layer_mu", kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * (i + 1) + 2)), bias_initializer=RandomUniform(minval=-s, maxval=s, seed=update_seed( seed, 2 * (i + 1) + 3)), kernel_regularizer=l2(l2reg), bias_regularizer=l2(l2reg), )(l) x_output = mu_output else: raise NotImplementedError( "Loss {} is not implemented yet for deep ensembles.". format(loss)) ensemble[modelname] = Model(inputs=[x_input], outputs=x_output) if verbose > 0: print(ensemble[modelname].summary()) seed = update_seed( seed, 2 * (i + 2) + 3 + 1 ) # update seed for each model in ensemble to achieve diversity print() self.models[ensemble_key] = ensemble