def single_layer_network_grid_cv(name, cv=5, save=True):
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(X_train, X_test)
param_grid = {
'shape': [(X_train.shape[1], )],
'neurons': np.arange(230),
'batch_size': [1028],
'epochs': [50],
'reg': np.geomspace(1e-4, 2, 25),
'if_reg': [True],
'shuffle': [True]
}
# param_grid = {'shape' : [(X_train.shape[1],)],
# 'neurons': np.arange(20,275,5),
# 'batch_size': [1028],
# 'epochs': [50],
# 'learning_rate': np.linspace(0.1,1,50),
# 'reg': np.geomspace(1e-8, 2, 50),
# 'if_reg': [True],
# 'shuffle': [False, True]}
def single_layer_network(shape, learning_rate, reg, if_reg):
"""
"""
net = Sequential()
if if_reg:
net.add(
Dense(45,
activation='relu',
input_shape=shape,
kernel_regularizer=regularizers.l2(reg)))
else:
net.add(Dense(45, activation='relu', input_shape=shape))
net.add(Dense(1, activation='linear'))
optimizer = Adam(learning_rate=learning_rate)
net.compile(optimizer=optimizer, loss='mean_squared_error')
return net
net = KerasRegressor(build_fn=single_layer_network, verbose=0)
net_cv = GridSearchCV(estimator=net,
param_grid=param_grid,
n_jobs=-1,
pre_dispatch=16,
refit=True,
cv=cv,
scoring='neg_mean_absolute_error').fit(
X_train, y_train)
display_name = ds.get_names()[name]
performance = metrics.apply_metrics(
'{} Single Layer Neural Network'.format(display_name), y_test,
net_cv.predict(X_test), y_train)
performance['Tuning Parameters'] = [net_cv.best_params_]
return net_cv
def support_vector_machine(name, cv=5):
"""
"""
display_name = ds.get_names()[name]
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(X_train, X_test)
to_score, scoring = metrics.create_metrics()
param_grid = {
'epsilon': np.linspace(-2, 2, 4),
'fit_intercept': [True],
'C': np.linspace(1e6, 1e10, 50),
'loss': ['epsilon_insensitive', 'squared_epsilon_insensitive'],
'dual': [False],
'random_state': [18]
}
model = LinearSVR()
model_cv = GridSearchCV(model,
param_grid=param_grid,
scoring='neg_mean_absolute_error',
n_jobs=-1,
pre_dispatch=16,
cv=cv,
refit=True).fit(X_train, y_train)
performance = metrics.apply_metrics(
'{} Linear Support Vector Machine'.format(display_name), y_test,
model_cv.predict(X_test), y_train)
performance['Tuning Parameters'] = [model_cv.best_params_]
return model_cv, performance
def get_baselines(save=False):
'''
'''
names = ['dataset_1', 'dataset_2', 'dataset_3']
models = [
RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor,
ExtraTreesRegressor
]
model_names = [
'random_forest', 'adaboost', 'gradient_boosting', 'extra_trees'
]
model_dict = dict(zip(models, model_names))
display_name = ds.get_names()
results = pd.DataFrame()
for name in names:
X_train, X_test, y_train, y_test, train = split.split_subset(name)
disp_name = display_name[name]
for func in models:
preds = func().fit(X_train, y_train).predict(X_test)
performance = metrics.apply_metrics(
'{} {}'.format(disp_name, model_dict[func]), y_test, preds)
results = pd.concat([results, performance], axis=0)
if save:
to_save = Path().resolve().joinpath('models', 'ensemble',
'{}.csv'.format('baseline_all'))
results.to_csv(to_save)
return results
def single_layer_network_randomized_cv(name, n_iter=20, cv=5, save=True):
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(X_train, X_test)
param_grid = {
'shape': [(X_train.shape[1], )],
'batch_size': [256, 512, 1028, 2056],
'epochs': [25, 50],
'learning_rate': [1e-4, 1e-3, 1e-2, 1e-1, 1, 10]
}
# 'reg': np.linspace(1e-4, 750, 500),
# 'if_reg': [True],
# 'shuffle': [False,True]}
#kernel_regularizer=regularizers.l2(reg))
def single_layer_network(shape, learning_rate):
"""
"""
net = Sequential()
net.add(Dense(45, activation='relu', input_shape=shape))
net.add(Dense(1, activation='linear'))
optimizer = Adam(learning_rate=learning_rate)
net.compile(optimizer=optimizer, loss='mean_squared_error')
return net
net = KerasRegressor(build_fn=single_layer_network,
verbose=0,
workers=8,
use_multiprocessing=True)
net_cv = RandomizedSearchCV(estimator=net,
param_distributions=param_grid,
n_jobs=-1,
pre_dispatch=16,
refit=True,
cv=cv,
scoring='neg_mean_absolute_error',
n_iter=n_iter,
random_state=18).fit(X_train, y_train)
display_name = ds.get_names()[name]
performance = metrics.apply_metrics(
'{} Single Layer Neural Network'.format(display_name), y_test,
net_cv.predict(X_test), y_train)
performance['Tuning Parameters'] = [net_cv.best_params_]
if save:
to_save_cv = Path().resolve().joinpath('models',
'cross_validation_outcomes',
'neural_network',
'{}.csv'.format(name))
results = pd.DataFrame.from_dict(net_cv.cv_results_)
results.to_csv(to_save_cv)
to_save_perf = Path().resolve().joinpath(
'models', 'neural_network', '{}_performance.csv'.format(name))
performance.to_csv(to_save_perf)
return net_cv, performance
def huber(name, cv=5):
'''
'''
display_name = ds.get_names()[name]
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(X_train, X_test)
to_score, scoring = metrics.create_metrics()
param_grid = {
'epsilon': np.linspace(1 + 1e-15, 1.2, 10),
'alpha': np.linspace(1e-8, 2, 10)
}
model = HuberRegressor()
model_cv = GridSearchCV(model,
param_grid=param_grid,
scoring='neg_mean_absolute_error',
n_jobs=-1,
pre_dispatch=16,
cv=cv,
refit=True).fit(X_train, y_train)
performance = metrics.apply_metrics('{} Huber'.format(display_name),
y_test, model_cv.predict(X_test),
y_train)
performance['Tuning Parameters'] = [model_cv.best_params_]
return model_cv, performance
def elastic_net(name, cv=5):
'''Outputs a fitted Elastic Net Regression Model with tuning parameters found through cross validation.
Inputs must be standardized.
l1_ratios are spread out on a log scale as recommended by package authors.
Number of folds in cross validation is by default 5.
n_jobs = -1 allows for all local processors to be utilized.
# '''
# if np.any(X_train.mean(axis = 0) > 1):
# raise ValueError('Numerical features must be standardized')
display_name = ds.get_names()[name]
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(X_train, X_test)
l1_ratios = np.geomspace(1e-8, 1, 50)
model = ElasticNetCV(l1_ratio=l1_ratios,
n_alphas=50,
cv=5,
verbose=0,
n_jobs=-1,
random_state=18).fit(X_train, y_train)
performance = metrics.apply_metrics('{} Elastic Net'.format(display_name),
y_test, model.predict(X_test), y_train)
performance['Tuning Parameters'] = [{
'Alpha': model.alpha_,
'L1 Ratio': model.l1_ratio_
}]
return model, performance
def baseline_creation(save=False):
'''
'''
file_names = ['dataset_1', 'dataset_2', 'dataset_3']
names = ['Dataset 1', "Dataset 2", "Dataset 3"]
sets = split.split_subsets(file_names)
df_avgs = pd.DataFrame()
df_ols = pd.DataFrame()
for i, file_name in enumerate(file_names):
y_bar = np.mean(sets[file_name][2])
preds = np.ones(len(sets[file_name][3])) * y_bar
avg_score = metrics.apply_metrics('{} Average'.format(names[i]),
sets[file_name][3], preds,
sets[file_name][2])
df_avgs = pd.concat([df_avgs, avg_score], axis=0)
ols_score = linear(file_name)[1]
df_ols = pd.concat([df_ols, ols_score], axis=0)
if save == True:
to_save_avgs = Path().resolve().joinpath('models', 'baseline',
'{}.csv'.format('averages'))
df_avgs.to_csv(to_save_avgs)
to_save_ols = Path().resolve().joinpath('models', 'baseline',
'{}.csv'.format('OLS'))
df_ols.to_csv(to_save_ols)
return df_avgs, df_ols
def linear(name):
'''Outputs a fitted Linear Regression Model.
Inputs can be standardized or not
'''
display_name = ds.get_names()[name]
X_train, X_test, y_train, y_test, train = split.split_subset(name, True)
model = LinearRegression().fit(X_train, y_train)
performance = metrics.apply_metrics('{} OLS'.format(display_name), y_test,
model.predict(X_test), y_train)
performance['Tuning Parameters'] = ""
return model, performance
def ridge(name, cv=5):
'''Outputs a fitted Ridge Regression Model with a penalty term tuned through cross validation.
'''
display_name = ds.get_names()[name]
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(X_train, X_test)
alphas = np.linspace(500, 750, 50)
model = RidgeCV(alphas=alphas, fit_intercept=True,
cv=cv).fit(X_train, y_train)
performance = metrics.apply_metrics('{} Ridge'.format(display_name),
y_test, model.predict(X_test), y_train)
performance['Tuning Parameters'] = [{'Alpha': model.alpha_}]
return model, performance
def k_neighbors_grid_cv(name, cv=5, save=True):
'''Conducts a grid search over all possible combinations of given parameters and returns result.
Uses parameters closely clustered around the best randomized search results.
Also returns back best fitted model by specified criteria (MAE).
'''
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(name, X_train, X_test)
to_score = metrics.create_metrics()[0]
param_grid = {
'n_neighbors': np.arange(20, 51, 2, dtype=int),
'weights': ['distance'],
'leaf_size': [8, 16, 128, 256]
}
model = KNeighborsRegressor(n_jobs=-1)
model_cv = GridSearchCV(n_jobs=-1,
estimator=model,
param_grid=param_grid,
scoring=to_score,
pre_dispatch=16,
refit=False,
cv=cv).fit(X_train, y_train)
display_name = ds.get_names()[name]
performance = pd.DataFrame()
variations = linear.get_model_variants(KNeighborsRegressor, model_cv)
for variation in variations:
model = variation.fit(X_train, y_train).predict(X_test)
performance = pd.concat([
performance,
metrics.apply_metrics('{} K Neighbors'.format(display_name),
y_test, model)
],
axis=0)
if save:
to_save = Path().resolve().joinpath('models',
'cross_validation_outcomes',
'other', 'k_neighbors',
'{}.csv'.format('grid'))
results = pd.DataFrame.from_dict(model_cv.cv_results_)
results.to_csv(to_save)
return model_cv, performance
def svr_grid_cv(name, standardize=False, cv=5):
"""
"""
if cv == 5:
cv_type = 'K-Fold'
else:
cv_type = "Time Series Split"
X_train, X_test, y_train, y_test, train = split.split_subset(name)
if standardize:
X_train = split.standardize(name, X_train)
X_test = split.standardize(name, X_test)
to_score = metrics.create_metrics()[0]
param_grid = {
'kernel': ['poly', 'rbf', 'sigmoid'],
'degree': np.arange(3, 9),
'gamma': ['scale', 'auto'],
'C': [2e-5, 2e-3, 2e-1, 2e1, 2e3, 2e5, 2e7, 2e9, 2e11]
}
model = SVR(n_jobs=-1)
model_cv = GridSearchCV(estimator=model,
param_distributions=param_grid,
n_jobs=-1,
pre_dispatch=16,
refit=False,
cv=cv,
scoring=to_score).fit(X_train, y_train)
display_name = ds.get_names()[name]
performance = pd.DataFrame()
variations = linear.get_model_variants(KNeighborsRegressor, model_cv)
for variation in variations:
model = variation.fit(X_train, y_train).predict(X_test)
performance = pd.concat([
performance,
metrics.apply_metrics(
'{} {} Support Vector Machine'.format(display_name, cv_type),
y_test, model)
],
axis=0)
return model_cv, performance
def extra_trees_grid_cv(name, cv=5, save=True):
"""
"""
X_train, X_test, y_train, y_test, train = split.split_subset(name)
extra_trees = ExtraTreesRegressor(n_jobs=-1,
random_state=18,
max_features=None,
bootstrap=False)
param_grid = {
'n_estimators': [250],
'max_depth': [20, 35],
'bootstrap': [True, False],
'max_features': [30, 45, 80],
'min_samples_split': [2, 8, 16],
'min_samples_leaf': [1, 2]
}
extra_trees_cv = GridSearchCV(n_jobs=-1,
estimator=extra_trees,
param_grid=param_grid,
pre_dispatch=16,
cv=cv,
refit=True,
scoring='neg_mean_absolute_error').fit(
X_train, y_train)
display_name = ds.get_names()[name]
performance = metrics.apply_metrics('{} Extra Trees'.format(display_name),
y_test, extra_trees_cv.predict(X_test),
y_train)
performance['Tuning Parameters'] = [extra_trees_cv.best_params_]
if save:
to_save = Path().resolve().joinpath('models',
'cross_validation_outcomes',
'ensemble', 'extra_trees',
'{}.csv'.format('grid'))
results = pd.DataFrame.from_dict(extra_trees_cv.cv_results_)
results.to_csv(to_save)
return extra_trees_cv, performance
def gradient_boosting_grid_cv(name, cv=5, save=True):
"""Conducts a grid search over all possible combinations of given parameters and returns the result
Uses parameters closely clustered around the best randomized search results.
Also returns back best fitted model by specified criteria (MAE).
"""
X_train, X_test, y_train, y_test, train = split.split_subset(name)
param_grid = {
'loss': ['ls', 'lad', 'huber'],
'learning_rate': np.geomspace(1e-6, 0.1, 5),
'n_estimators': [900],
'min_samples_split': [4, 64, 128, 256],
'min_samples_leaf': [8, 128],
'max_depth': [4, 5, 15],
'alpha': np.linspace(0.1, 1, 5),
'max_features': [3, 40, 60]
}
gradient_boosting = GradientBoostingRegressor(random_state=18)
gradient_boosting_cv = GridSearchCV(n_jobs=-1,
estimator=gradient_boosting,
param_grid=param_grid,
cv=cv,
refit=True,
scoring='neg_mean_absolute_error',
pre_dispatch=16).fit(X_train, y_train)
display_name = ds.get_names()[name]
performance = metrics.apply_metrics(
'{} Gradient Boosting'.format(display_name), y_test,
gradient_boosting_cv.predict(X_test), y_train)
performance['Tuning Parameters'] = [gradient_boosting_cv.best_params_]
if save:
to_save = Path().resolve().joinpath('models',
'cross_validation_outcomes',
'ensemble', 'gradient_boosting',
'{}.csv'.format('grid'))
results = pd.DataFrame.from_dict(gradient_boosting_cv.cv_results_)
results.to_csv(to_save)
return gradient_boosting_cv, performance
def gradient_boosting_randomized_cv(name, n_iter=50, cv=5, save=True):
"""Conducts a randomized search of cross validation for given parameters of Gradient Boosting and returns results.
"""
X_train, X_test, y_train, y_test, train = split.split_subset(name)
param_grid = {
'loss': ['ls', 'lad', 'huber'],
'learning_rate': np.append(np.array([0]), np.geomspace(1e-6, 1, 50)),
'n_estimators': np.linspace(500, 1000, 50, dtype=int),
'min_samples_split': [2, 4, 8, 16, 32, 64, 128, 256],
'min_samples_leaf': [1, 2, 4, 8, 16, 32, 64, 128],
'max_depth': [2, 3, 4, 5, 10, 15],
'alpha': np.linspace(1e-6, 1, 25),
'max_features': np.linspace(2, len(X_train.columns), num=50,
dtype=int),
}
gradient_boosting = GradientBoostingRegressor()
gradient_boosting_cv = RandomizedSearchCV(
estimator=gradient_boosting,
n_jobs=-1,
pre_dispatch=16,
param_distributions=param_grid,
n_iter=n_iter,
cv=cv,
refit=True,
scoring='neg_mean_absolute_error').fit(X_train, y_train)
display_name = ds.get_names()[name]
performance = metrics.apply_metrics(
'{} Gradient Boosting'.format(display_name), y_test,
gradient_boosting_cv.predict(X_test), y_train)
performance['Tuning Parameters'] = [gradient_boosting_cv.best_params_]
if save:
to_save = Path().resolve().joinpath('models',
'cross_validation_outcomes',
'ensemble', 'gradient_boosting',
'{}.csv'.format('randomized'))
results = pd.DataFrame.from_dict(gradient_boosting_cv.cv_results_)
results.to_csv(to_save)
return gradient_boosting_cv, performance
def pca_cv(name, save=False):
'''
'''
display_name = ds.get_names()[name]
X_train, X_test, y_train, y_test, train = split.split_subset(name)
num_numerical = ds.get_number_numerical()[name]
X_train_s, X_test_s = split.standardize(name, X_train, X_test)
X_train_s_numerical = X_train_s.iloc[:, 0:num_numerical]
X_train_s_categorical = X_train_s.iloc[:, num_numerical:]
X_test_s_numerical = X_test_s.iloc[:, 0:num_numerical]
X_test_s_categorical = X_test_s.iloc[:, num_numerical:]
df = pd.DataFrame()
ols = LinearRegression()
ev = []
for i in np.arange(1, num_numerical):
pca = PCA(i, random_state=18)
X_train_s_numerical_reduced = pd.DataFrame(
pca.fit_transform(X_train_s_numerical),
index=X_train_s_categorical.index)
X_test_s_numerical_reduced = pd.DataFrame(
pca.transform(X_test_s_numerical),
index=X_test_s_categorical.index)
X_train_s = pd.concat(
[X_train_s_numerical_reduced, X_train_s_categorical], axis=1)
X_test_s = pd.concat(
[X_test_s_numerical_reduced, X_test_s_categorical], axis=1)
model = ols.fit(X_train_s, y_train)
preds = model.predict(X_test_s)
preds = metrics.apply_metrics(
'{}: {} dimensions'.format(display_name, i), y_test, preds.ravel(),
y_train)
df = pd.concat([df, preds], axis=0)
ev.append(1 - sum(pca.explained_variance_))
if save:
to_save = Path().resolve().joinpath('features', 'pca',
'{}.csv'.format(name))
df.to_csv(to_save)
return df, ev
def random_forest_grid_cv(name, cv=5, save=True):
'''Conducts a grid search over all possible combinations of given parameters and returns result.
Uses parameters closely clustered around the best randomized search results.
Also returns back best fitted model by specified criteria (MAE).
'''
display_name = ds.get_names()[name]
X_train, X_test, y_train, y_test, train = split.split_subset(name)
param_grid = {
'n_estimators': [400],
'max_depth': [30, 50, 90],
'bootstrap': [False],
'max_features': [30, 40, 50],
'min_samples_split': [4],
'min_samples_leaf': [1, 2, 8]
}
rf = RandomForestRegressor()
rf_cv = GridSearchCV(n_jobs=-1,
estimator=rf,
param_grid=param_grid,
scoring='neg_mean_absolute_error',
pre_dispatch=16,
refit=True,
cv=cv).fit(X_train, y_train)
performance = metrics.apply_metrics(
'{} Random Forest'.format(display_name), y_test, rf_cv.predict(X_test),
y_train)
performance['Tuning Parameters'] = [rf_cv.best_params_]
if save:
to_save = Path().resolve().joinpath('models',
'cross_validation_outcomes',
'ensemble', 'random_forest',
'{}.csv'.format('grid'))
results = pd.DataFrame.from_dict(rf_cv.cv_results_)
results.to_csv(to_save)
return rf_cv, performance
def adaboost_grid_cv(name, cv=5, save=True):
'''Conducts a grid search over all possible combinations of given parameters and returns the result.
Uses parameters closely clustered around the best randomized search results.
'''
X_train, X_test, y_train, y_test, train = split.split_subset(name)
param_grid = {
'base_estimator': [DecisionTreeRegressor(max_depth=5)],
'n_estimators': [250],
'loss': ['linear', 'exponential'],
'learning_rate': np.geomspace(1e-6, 0.2, 20)
}
adaboost = AdaBoostRegressor()
adaboost_cv = GridSearchCV(estimator=adaboost,
param_grid=param_grid,
scoring='neg_mean_absolute_error',
refit=True,
cv=cv,
n_jobs=-1,
pre_dispatch=16).fit(X_train, y_train)
display_name = ds.get_names()[name]
performance = metrics.apply_metrics('{} AdaBoost'.format(display_name),
y_test, adaboost_cv.predict(X_test),
y_train)
performance['Tuning Parameters'] = [adaboost_cv.best_params_]
if save:
to_save = Path().resolve().joinpath('models',
'cross_validation_outcomes',
'ensemble', 'adaboost',
'{}.csv'.format('grid'))
results = pd.DataFrame.from_dict(adaboost_cv.cv_results_)
results.to_csv(to_save)
return adaboost_cv, performance
def lasso(name, cv=5):
'''Outputs a fitted Lasso Regression Model with a penalty term tuned through cross validation.
Inputs must be standardized.
Number of folds in cross validation is by default 5.
n_jobs = -1 allows for all local processors to be utilized.
'''
display_name = ds.get_names()[name]
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(X_train, X_test)
model = LassoCV(n_alphas=50,
verbose=0,
cv=5,
n_jobs=-1,
copy_X=True,
tol=1e-3,
random_state=18).fit(X_train, y_train)
performance = metrics.apply_metrics('{} Lasso'.format(display_name),
y_test, model.predict(X_test), y_train)
performance['Tuning Parameters'] = [{'Alpha': model.alpha_}]
params = model.coef_
return model, performance, params
def multi_layer_network_performance(name, save=True):
"""
"""
X_train, X_test, y_train, y_test, train = split.split_subset(name)
X_train, X_test = split.standardize(name, X_train, X_test)
def multi_layer_network(neurons_l1,
neurons_ol,
num_layers,
shape,
learning_rate,
reg_l1,
reg_ol,
epochs,
batch_size,
X_train=X_train,
y_train=y_train):
"""
"""
net = Sequential()
net.add(
Dense(neurons_l1,
activation='relu',
input_shape=shape,
kernel_regularizer=regularizers.l2(reg_l1)))
for i in np.arange(num_layers):
net.add(
Dense(neurons_ol[i],
activation='relu',
input_shape=shape,
kernel_regularizer=regularizers.l2(reg_ol[i])))
net.add(Dense(1, activation='linear'))
optimizer = Adam(learning_rate=learning_rate)
net.compile(optimizer=optimizer, loss='mean_squared_error')
net.fit(X_train,
y_train,
epochs=epochs,
batch_size=batch_size,
workers=7,
use_multiprocessing=True)
return net
results = pd.read_csv(Path().resolve().joinpath(
'models', 'cross_validation_outcomes', 'neural_network',
'{}_{}_{}.csv'.format(name, 'multi_layer_network', 'randomized')),
index_col=0)
bestr2 = results.loc[results['rank_test_$R^2$'] == 1, 'params'].values[0]
bestmae = results.loc[results['rank_test_Mean Absolute Error'] == 1,
'params'].values[0]
bestrmse = results.loc[results['rank_test_Root Mean Square Error'] == 1,
'params'].values[0]
display_name = ds.get_names()[name]
dict_list = [bestr2, bestmae, bestrmse]
unique_dict_list = [
dict(t) for t in {tuple(sorted(d.items()))
for d in dict_list}
]
performance = pd.DataFrame()
for item in unique_dict_list:
preds = multi_layer_network(**item).predict(X_test)
performance = pd.concat([
performance,
metrics.apply_metrics('{} Single Layer NN'.format(display_name),
y_test, preds)
],
axis=0)
if save:
to_save = Path().resolve().joinpath(
'models', 'neural_network', '{}.csv'.format('multi_layer_network'))
performance.to_csv(to_save)