def compile_models(self, verbose: int = 0) -> NoReturn:
"""Compiles the neural network architectures.
Arguments
----------
verbose:
Verbosity level.
"""
print("\nCompile the following NOMU_DJ Models:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
self.compile_mean_models()
self.compile_sigma_models()
# update all parameters of main architectures
self.update_all_weights()
print()
def fit_models(
self,
x: np.array,
y: np.array,
verbose: int = 0,
) -> NoReturn:
"""Fits the neural network architectures to specified data.
Arguments
----------
x :
input data (features)
y :
output data (targets).
verbose :
Level of verbosity.
"""
print("\nFit the following MC Dropout Models:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key in self.models.keys():
print(key)
p = self.parameters[key]
model = self.models[key]
# fit model
start = datetime.now()
history = model.fit(
x, y, epochs=p["epochs"], batch_size=p["batch_size"], verbose=verbose
)
end = datetime.now()
diff = end - start
print(
"Elapsed: {}d {}h:{}m:{}s".format(*timediff_d_h_m_s(diff)),
"(" + datetime.now().strftime("%H:%M %d-%m-%Y") + ")",
)
self.models[key] = model
if self.histories[key] is None:
self.histories[key] = history.history
else:
self.histories[key] = {
loss_key: loss_value + history.history[loss_key]
for loss_key, loss_value in self.histories[key].items()
}
self.parameters[key]["actual_epochs"] = len(self.histories[key]["loss"])
print()
def compile_models(self, verbose: int = 0) -> NoReturn:
"""Compiles the neural network architectures.
Arguments
----------
verbose:
Verbosity level.
"""
print("\nCompile the following MC Dropout Models:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key in self.models.keys():
print(key)
p = self.parameters[key]
model = self.models[key]
# compile model
# tensorflow version 2.1
# (i) use adam
if p["optimizer"] == "Adam":
optimizer = Adam(
learning_rate=p["learning_rate"],
beta_1=p["beta_1"],
beta_2=p["beta_2"],
epsilon=p["epsilon"],
amsgrad=p["amsgrad"],
name=p["optimizer"],
clipnorm=p["clipnorm"],
)
if p["optimizer"] == "SGD":
# (ii) plain vanilla gd
optimizer = SGD(
learning_rate=p["learning_rate"],
momentum=p["momentum"],
nesterov=p["nesterov"],
name=p["optimizer"],
clipnorm=p["clipnorm"],
)
model.compile(
optimizer=optimizer, loss=p["loss"], experimental_run_tf_function=False
)
self.models[key] = model
print()
def fit_models(
self,
x: np.array,
y: np.array,
verbose: int = 0,
x_min_aug: float = -1 - 0.1,
x_max_aug: float = 1 + 0.1,
) -> NoReturn:
"""Fits the neural network architectures to specified data.
Arguments
----------
x :
input data (features)
y :
output data (targets).
verbose :
Level of verbosity.
x_min_aug :
Min of box bound for augmented data points.
x_max_aug :
Max of box bound for augmented data points.
"""
# fit NOMU_dj models
print("\nFit the following NOMU_DJ Models:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
# fit mean models
self.fit_mean_models(x=x, y=y, verbose=verbose)
# set parameters of main architecture in sigma model to the ones of the mean model
self.update_sigma_weights()
# fit sigma model
self.fit_sigma_models(x=x,
y=y,
verbose=verbose,
x_min_aug=x_min_aug,
x_max_aug=x_max_aug)
# update all parameters of main architectures
self.update_all_weights()
print()
def initialize_models(self, verbose: int = 0) -> NoReturn:
"""Initializes the kernels for the GPs.
Arguments
----------
verbose :
Verbosity level.
"""
print("\nInitialize the following Gaussian Processes:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key in self.models.keys():
print(key)
p = self.parameters[key]
if p["kernel"] == "rbf":
if p["whitekernel"]:
K = C(
constant_value=p["constant_value"],
constant_value_bounds=p["constant_value_bounds"],
) * RBF(
length_scale=p["length_scale"],
length_scale_bounds=p["length_scale_bounds"],
) + WhiteKernel(
noise_level=p["noise_level"],
noise_level_bounds=p["noise_level_bounds"],
)
else:
K = C(
constant_value=p["constant_value"],
constant_value_bounds=p["constant_value_bounds"],
) * RBF(
length_scale=p["length_scale"],
length_scale_bounds=p["length_scale_bounds"],
)
else:
raise NotImplementedError("Kernel {} not implemented".format(
p["kernel"]))
self.initial_kernels[key] = K
print()
def fit_models(self,
x: np.array,
y: np.array,
verbose: int = 0) -> NoReturn:
"""Fits the GPs to specified data.
Arguments
----------
x :
input data (features)
y :
output data (targets).
verbose :
Level of verbosity.
"""
print("\nFit the following Gaussian Processes:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key, model in self.models.items():
print(key)
start = datetime.now()
self.models[key].fit(x, y)
end = datetime.now()
diff = end - start
print(
"Elapsed: {}d {}h:{}m:{}s".format(*timediff_d_h_m_s(diff)),
"(" + datetime.now().strftime("%H:%M %d-%m-%Y") + ")",
)
print()
def compile_models(self, verbose: int = 0) -> NoReturn:
"""Compiles the GPs.
Arguments
----------
verbose:
Verbosity level.
"""
print("\nCompile the following Gaussian Processes:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key, kernel in self.initial_kernels.items():
print(key)
p = self.parameters[key]
gp = GaussianProcessRegressor(
kernel=kernel,
alpha=p["alpha"],
optimizer=p["optimizer"],
n_restarts_optimizer=p["n_restarts_optimizer"],
normalize_y=p["normalize_y"],
copy_X_train=p["copy_X_train"],
random_state=p["random_state"],
)
self.models[key] = gp
print()
def initialize_models(
self, s: float = 0.05, activation: str = "relu", verbose: int = 0
) -> NoReturn:
"""Initializes the neural network architectures.
Arguments
----------
s :
Interval parameters for uniform random weight initialization in the intervall [-s,s]
activation :
Tf activation function.
verbose :
Verbosity level.
"""
print("\nInitialize the following MC Dropout Models:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key in self.models.keys():
print(key)
p = self.parameters[key]
layers = p["layers"]
l2reg = p["l2reg"]
seed = p["seed_init"]
dropout = p["dropout_prob"]
# input layer
x_input = Input(shape=(layers[0],), name="input_layer")
# first hidden layer
l = Dense(
layers[1],
activation=activation,
name="hidden_layer_{}".format(1),
kernel_initializer=RandomUniform(minval=-s, maxval=s, seed=seed),
bias_initializer=RandomUniform(
minval=-s, maxval=s, seed=update_seed(seed, 1)
),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(x_input)
if dropout != 0:
l = Dropout(dropout)(l, training=True)
# hidden layers
for i, n in enumerate(layers[2:-1]):
l = Dense(
n,
activation=activation,
name="hidden_layer_{}".format(i + 2),
kernel_initializer=RandomUniform(
minval=-s, maxval=s, seed=update_seed(seed, 2 * i + 2)
),
bias_initializer=RandomUniform(
minval=-s, maxval=s, seed=update_seed(seed, 2 * i + 3)
),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(l)
if dropout != 0:
l = Dropout(dropout)(l, training=True)
# output layer
x_output = Dense(
layers[-1],
activation="linear",
name="output_layer",
kernel_initializer=RandomUniform(
minval=-s, maxval=s, seed=update_seed(seed, -1)
),
bias_initializer=RandomUniform(
minval=-s, maxval=s, seed=update_seed(seed, -2)
),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(l)
self.models[key] = Model(inputs=[x_input], outputs=x_output)
if verbose > 0:
print("Architecture\n")
print(self.models[key].summary())
print()
def fit_models(
self,
x: np.array,
y: np.array,
verbose: int = 0,
x_min_aug: float = -1 - 0.1,
x_max_aug: float = 1 + 0.1,
) -> NoReturn:
"""Fits the neural network architectures to specified data.
Arguments
----------
x :
input data (features)
y :
output data (targets).
verbose :
Level of verbosity.
x_min_aug :
Min of box bound for augmented data points.
x_max_aug :
Max of box bound for augmented data points.
"""
# fit NOMU models
print("\nFit the following NOMU Models:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key in self.models.keys():
print(key)
p = self.parameters[key]
model = self.models[key]
# set up generator
generator = DataGenerator(
batch_size=p["batch_size"],
x=x,
y=y,
n_train=p["n_train"],
n_aug=p["n_aug"],
MCaug=p["MCaug"],
x_min_aug=x_min_aug,
x_max_aug=x_max_aug,
joint_loss=True,
)
# fit model
best_weights_callback = ReturnBestWeights(monitor="loss",
verbose=1,
mode="min",
baseline=None)
start = datetime.now()
history = model.fit(
x=generator,
epochs=p["epochs"],
verbose=verbose,
callbacks=[
best_weights_callback,
PredictionHistory(generator)
],
)
end = datetime.now()
diff = end - start
print(
"Elapsed: {}d {}h:{}m:{}s".format(*timediff_d_h_m_s(diff)),
"(" + datetime.now().strftime("%H:%M %d-%m-%Y") + ")",
)
self.models[key] = model
if self.histories[key] is None:
self.histories[key] = history.history
else:
self.histories[key] = {
loss_key: loss_value + history.history[loss_key]
for loss_key, loss_value in self.histories[key].items()
}
self.parameters[key]["actual_epochs"] = len(
self.histories[key]["loss"])
print()
def compile_models(self, verbose: int = 0) -> NoReturn:
"""Compiles the neural network architectures.
Arguments
----------
verbose:
Verbosity level.
"""
print("\nCompile the following NOMU Models:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key in self.models.keys():
print(key)
p = self.parameters[key]
model = self.models[key]
flag = self.flags[key]
# define loss
custom_loss = [
squared_loss_wrapper(flag=flag,
n_train=p["n_train"],
n_aug=p["n_aug"]),
r_loss_wrapper(
flag=flag,
mu_sqr=p["mu_sqr"],
mu_exp=p["mu_exp"],
c_exp=p["c_exp"],
n_train=p["n_train"],
n_aug=p["n_aug"],
stable_aug_loss=p["stable_aug_loss"],
c_2=p["c_sqr_stable_aug_loss"],
),
]
# compile model
# tensorflow version 2.1
# (i) use adam
if p["optimizer"] == "Adam":
optimizer = Adam(
learning_rate=p["learning_rate"],
beta_1=p["beta_1"],
beta_2=p["beta_2"],
epsilon=p["epsilon"],
amsgrad=p["amsgrad"],
name=p["optimizer"],
clipnorm=p["clipnorm"],
)
if p["optimizer"] == "SGD":
# (ii) plain vanilla gd
optimizer = SGD(
learning_rate=p["learning_rate"],
momentum=p["momentum"],
nesterov=p["nesterov"],
name=p["optimizer"],
clipnorm=p["clipnorm"],
)
model.compile(
optimizer=optimizer,
loss=custom_loss,
experimental_run_tf_function=False,
)
self.models[key] = model
print()
def initialize_models(self,
s: float = 0.05,
activation: str = "relu",
verbose: int = 0) -> NoReturn:
"""Initializes the neural network architectures.
Arguments
----------
s :
Interval parameters for uniform random weight initialization in the intervall [-s,s]
activation :
Tf activation function.
verbose :
Verbosity level.
"""
print("\nInitialize the following NOMU Models:")
print(
"**************************************************************************"
)
if verbose > 0:
for key in self.models.keys():
print(key)
pretty_print_dict(self.parameters[key])
print()
print(
"**************************************************************************"
)
for key in self.models.keys():
print(key)
p = self.parameters[key]
layers = p["layers"]
side_layers = p["side_layers"]
l2reg = p["l2reg"]
l2reg_sig = p["l2reg_sig"]
seed = p["seed_init"]
RSN = p["RSN"]
dropout = p["dropout_prob"]
# input layer
x_input = Input(shape=(layers[0], ), name="input_layer")
flag = Input(shape=(1, ), name="flag_input")
# main architecture -------------------------------------------------------
# first hidden layer
y = Dense(
layers[1],
activation=activation,
name="hidden_layer_{}".format(1),
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=seed),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(seed, 1)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
trainable=not (RSN),
)(x_input)
if dropout != 0 and dropout is not None:
y = Dropout(dropout)(y, training=False)
# hidden layers
for i, n in enumerate(layers[2:-1]):
y = Dense(
n,
activation=activation,
name="hidden_layer_{}".format(i + 2),
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed, 2 * i + 2)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed, 2 * i + 3)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
trainable=not (RSN),
)(y)
if dropout != 0 and dropout is not None:
y = Dropout(dropout)(y, training=False)
# output layer
y_output = Dense(
layers[-1],
activation="linear",
name="output_layer",
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(seed, -1)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(seed, -2)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(y)
# side architecture r -----------------------------------------------------
# set new seed
seed = update_seed(seed, 2**6)
# first hidden layer
r = Dense(
side_layers[1],
activation=activation,
name="r_hidden_layer_{}".format(1),
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=seed),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(seed, 1)),
kernel_regularizer=l2(l2reg_sig),
bias_regularizer=l2(l2reg_sig),
trainable=not (RSN),
)(x_input)
# hidden layers
for i, n in enumerate(side_layers[2:-1]):
r = Dense(
n,
activation=activation,
name="r_hidden_layer_{}".format(i + 2),
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed, 2 * i + 2)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed, 2 * i + 3)),
kernel_regularizer=l2(l2reg_sig),
bias_regularizer=l2(l2reg_sig),
trainable=not (RSN),
)(r)
# concatenate last hidden of y and r
y_r_concat = concatenate([y, r])
r_output = Dense(
side_layers[-1],
activation="linear",
name="r_output_layer",
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(seed, -1)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(seed, -2)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(y_r_concat)
self.models[key] = Model(inputs=[x_input, flag],
outputs=[y_output, r_output])
self.flags[key] = flag
print()
def fit_models(
self,
x: np.array,
y: np.array,
verbose: int = 0,
) -> NoReturn:
"""Fits the neural network architectures to specified data.
Arguments
----------
x :
input data (features)
y :
output data (targets).
verbose :
Level of verbosity.
"""
print("\nFit the following Deep Ensembles:")
print(
"**************************************************************************"
)
if verbose > 0:
for ensemble_key in self.models.keys():
print(ensemble_key)
pretty_print_dict(self.parameters[ensemble_key])
print()
print(
"**************************************************************************"
)
for ensemble_key, ensemble in self.models.items():
print(ensemble_key)
p = self.parameters[ensemble_key]
start = datetime.now()
history = OrderedDict()
for model_key, model in ensemble.items():
print("Fit {}".format(model_key))
tmp = model.fit(
x,
y,
epochs=p["epochs"],
batch_size=p["batch_size"],
verbose=verbose,
callbacks=[PredictionHistory_DE(x, y)],
)
history[model_key] = tmp.history
end = datetime.now()
diff = end - start
print(
"Elapsed: {}d {}h:{}m:{}s".format(*timediff_d_h_m_s(diff)),
"(" + datetime.now().strftime("%H:%M %d-%m-%Y") + ")",
)
self.models[ensemble_key] = ensemble
if self.histories[ensemble_key] is None:
self.histories[ensemble_key] = history
else:
self.histories[ensemble_key] = {
model_key: {
loss_key: loss_value + history[model_key][loss_key]
for loss_key, loss_value in
self.histories[ensemble_key][model_key].items()
}
for model_key in self.models[ensemble_key].keys()
}
self.parameters[ensemble_key]["actual_epochs"] = len(
self.histories[ensemble_key][list(
self.histories[ensemble_key].keys())[0]]["loss"])
print()
print()
def compile_models(self, verbose: int = 0) -> NoReturn:
"""Compiles the neural network architectures.
Arguments
----------
verbose:
Verbosity level.
"""
print("\nCompile the following Deep Ensembles:")
print(
"**************************************************************************"
)
if verbose > 0:
for ensemble_key in self.models.keys():
print(ensemble_key)
pretty_print_dict(self.parameters[ensemble_key])
print()
print(
"**************************************************************************"
)
for ensemble_key in self.models.keys():
print(ensemble_key)
p = self.parameters[ensemble_key]
for model_key, model in self.models[ensemble_key].items():
print("Compile:", model_key)
if p["loss"] == "nll":
loss_DE = gaussian_nll
elif p["loss"] == "mse":
loss_DE = "mse"
else:
raise NotImplementedError(
"{} loss is not implemented.".format(p["loss"]))
# tensorflow version 2.1
if p["optimizer"] == "Adam":
optimizer = Adam(
learning_rate=p["learning_rate"],
beta_1=p["beta_1"],
beta_2=p["beta_2"],
epsilon=p["epsilon"],
amsgrad=p["amsgrad"],
name=p["optimizer"],
clipnorm=p["clipnorm"],
)
elif p["optimizer"] == "SGD":
optimizer = SGD(
learning_rate=p["learning_rate"],
momentum=p["momentum"],
nesterov=p["nesterov"],
name=p["optimizer"],
clipnorm=p["clipnorm"],
)
else:
raise NotImplementedError(
"{} optimizer is not implemented.".format(
p["optimizer"]))
model.compile(
optimizer=optimizer,
loss=loss_DE,
experimental_run_tf_function=False,
)
print()
def initialize_models(self,
s: float = 0.05,
activation: str = "relu",
verbose: int = 0) -> NoReturn:
"""Initializes the neural network architectures.
Arguments
----------
s :
Interval parameters for uniform random weight initialization in the intervall [-s,s]
activation :
Tf activation function.
verbose :
Verbosity level.
"""
print("\nInitialize the following Deep Ensembles:")
print(
"**************************************************************************"
)
if verbose > 0:
for ensemble_key in self.models.keys():
print(ensemble_key)
pretty_print_dict(self.parameters[ensemble_key])
print()
print(
"**************************************************************************"
)
for ensemble_key in self.models.keys():
print(ensemble_key)
p = self.parameters[ensemble_key]
layers = p["layers"]
l2reg = p["l2reg"]
number_of_networks = p["number_of_networks"]
loss = p["loss"]
softplus_min_var = p["softplus_min_var"]
l2reg = p["l2reg"]
seed = p["seed_init"]
ensemble = {}
for i in range(1, number_of_networks + 1):
modelname = "NN_{}".format(i)
print("Initialize:", modelname)
# input layer
x_input = Input(shape=(layers[0], ),
name=modelname + "_input_layer")
# first hidden layer
l = Dense(
layers[1],
activation=activation,
name=modelname + "_hidden_layer_{}".format(1),
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(seed,
0)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(seed, 1)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(x_input)
# hidden layers
for i, n in enumerate(layers[2:-1]):
l = Dense(
n,
activation=activation,
name=modelname + "_hidden_layer_{}".format(i + 2),
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed, 2 * i + 2)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed, 2 * i + 3)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(l)
if loss == "nll":
# output layer with two parameters for each output dimension in case loss==nll:
mu_output = Dense(
layers[-1],
activation="linear",
name=modelname + "_output_layer_mu",
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed,
2 * (i + 1) + 2)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed,
2 * (i + 1) + 3)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(l)
sigma_output = Dense(
layers[-1],
activation=softplus_wrapper(min_var=softplus_min_var),
name=modelname + "_output_layer_sigma",
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed,
2 * (i + 2) + 2)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed,
2 * (i + 2) + 3)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(l)
x_output = concatenate([mu_output, sigma_output])
elif loss == "mse":
# output layer with two parameters for each output dimension in case loss==nll:
mu_output = Dense(
layers[-1],
activation="linear",
name=modelname + "_output_layer_mu",
kernel_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed,
2 * (i + 1) + 2)),
bias_initializer=RandomUniform(minval=-s,
maxval=s,
seed=update_seed(
seed,
2 * (i + 1) + 3)),
kernel_regularizer=l2(l2reg),
bias_regularizer=l2(l2reg),
)(l)
x_output = mu_output
else:
raise NotImplementedError(
"Loss {} is not implemented yet for deep ensembles.".
format(loss))
ensemble[modelname] = Model(inputs=[x_input], outputs=x_output)
if verbose > 0:
print(ensemble[modelname].summary())
seed = update_seed(
seed, 2 * (i + 2) + 3 + 1
) # update seed for each model in ensemble to achieve diversity
print()
self.models[ensemble_key] = ensemble
