# combine the ensemble predictions into a single prediction for each trial of each hyperparam setting
#
predictions = []
for result in results:
predictor_results = result[2] #M* [(m_ij,s_ij)] each of len trials
if SOFTVOTING:
bagging_predictions = trials_soft_prediction(
predictor_results, Config.trials)
else:
bagging_predictions = trials_hard_prediction(
predictor_results, Config.trials)
predictions.append([result[0], result[1], bagging_predictions])
## compute the normalized error for all hyperparam runs & all trials
V_0 = generate_V_0(100000, Config.Delta, Config.d)
logger.info(f"V_0 = {V_0}")
normalized_error_results = []
for result in predictions:
errors = []
for index, prediction in enumerate(result[2]):
error = normalized_error_VT(prediction,
DataContainer.y_test_list[index], V_0)
errors.append(error)
normalized_error_results.append([result[0], result[1], errors])
print(normalized_error_results)
if WRITEBACK:
write_results("mpi_regularized_bagging", normalized_error_results,
Config)
"""
## generate Training set, Test set & V_0s
X_train, y_train = generate_train_set(Config.N_train, Config.Delta,
Config.d)
X_test, y_test = generate_test_set(Config.N_test, Config.Delta, Config.d)
V_0_train = generate_V_0(Config.N_train, Config.Delta, Config.d)
V_0_test = generate_V_0(Config.N_test, Config.Delta, Config.d)
logger.info(f"V_0_test = {V_0_test}")
reference = create_GPR(Config.N_train)
reference.fit(X_train, y_train)
f_X = reference.predict(X_test)
reference_error = normalized_error_VT(f_X, y_test, V_0_test)
logger.info(f"reference error : {reference_error}")
## MPI execute
results = []
with MPIPoolExecutor() as executor:
futures = []
# evaluate model for all points in grid by creating new mpi node
for r in Config.Ratios:
logger.info(f"starting evaluation for ratio {r}")
future = executor.submit(evaluate_boosting,
X_train.copy(),
y_train.copy(),
X_test.copy(),
y_test.copy(),
def evaluate_model(base_model,
hyperparams,
X_train,
y_train,
d,
DeltaT,
trials,
N_test,
samples_generator,
V_0=None,
logger=None,
random_seeds=None):
"""
Sets hyperparameters of the model, trains it first and evaluates it afterwards.
Used in GridSearch settings
:param base_model: model to used
:type base_model: sklearn model
:param hyperparams: dict of hyperparameters, containing their exact name as key
:type hyperparams: dict{str: value}
:param X_train: Training set
:type X_train: np.ndarray(N x (d*T))
:param y_train: labels of training set
:type y_train: np.ndarray( N x 1 )
:param d: number of stocks
:type d: integer
:param DeltaT: list of the delta Ts between the driver moments
:type DeltaT: list of length T
:param trials: number of times to evaluate the model
:type trials: integer
:param N_test: number of samples for each evaluation
:type N_test: integer
:param samples_generator: stochastic generator used to generate test samples
:type samples_generator: StochasticModelBase
:param V_0: V_0 value, could be obtained using larger set to improve accuracy of the results
:type V_0: optional float
:param logger: Optional, logger for debug information
:param seeds: Optional, list of seeds (in size of trials),
so that the same seed is used to generate the testset over multiple instances of the gridsearch
:return: list containing the normalized errrors
:rtype: list of size trials
"""
errors = []
try:
if logger:
logger.debug(
f" {hyperparams} -> thread id = {threading.current_thread().ident}"
)
else:
print(
f" {hyperparams} -> thread id = {threading.current_thread().ident}"
)
# create the model and set the hyperparams
model = clone(base_model)
for k, v in hyperparams.items():
setattr(model, k, v)
# train the model
model.fit(X_train, y_train)
# evaluate the model
for trial in range(trials):
if random_seeds:
np.random.seed(random_seeds[trial])
s_test = samples_generator(N_test, d, DeltaT)
s_test.generate_samples()
y_test = s_test.y
X_test = s_test.X
S_test = s_test.S
Flattened_X_test = flatten_X(X_test)
V_T = y_test
if V_0 is None:
V_0 = s_test.generate_true_V(0)
V_0 = V_0.mean()
y_hat = model.predict(Flattened_X_test)
error = normalized_error_VT(y_hat, V_T, V_0).item()
if logger:
logger.debug(f"{hyperparams} , {trial} -> {error}")
else:
print(f"{hyperparams} , {trial} -> {error}")
# add normalized error to the list
errors.append(error)
if logger:
logger.info(f"{hyperparams} -> {errors}")
else:
print(f"{hyperparams} -> {errors}")
return errors
except Exception as e:
if logger:
logger.warn(traceback.format_exc())
else:
traceback.print_exc()
return None
