def main():
c_pairs = [f_1,f_2,f_3]
kind = ['q_mse']
options = []
results = {}
for c_pair in c_pairs:
for opt in kind:
options.append((c_pair,opt))
for c_pair,opt in options:
print(c_pair,opt)
def obj_f(lag,drmse,krmse,window_size):
f_lag = int(lag)
f_drmse = np.exp(drmse)
f_krmse = np.exp(krmse)
f_window = int(window_size)
obj_f.m += 1
print(obj_f.m)
if (opt == 'mse'):
return trainMSE(c_pair+'.csv',f_lag,f_drmse,f_krmse,f_window)
elif (opt == 'profit'):
return trainProfit(c_pair+'.csv',f_lag,f_drmse,f_krmse,f_window)
elif (opt == 'q_mse'):
return trainMSEPredictLogRet(c_pair+'.csv',f_lag,f_drmse,f_krmse,f_window)
elif (opt == 'q_profit'):
return trainProfitPredictLogRet(c_pair+'.csv',f_lag,f_drmse,f_krmse,f_window)
obj_f.m = 0
kwargs = {'lag':[5,20],
'drmse':[np.log(0.001),np.log(0.01)],
'krmse':[np.log(0.001),np.log(0.01)],
'window_size':[10,300]}
ret = None
if (opt == 'mse'):
ret = optunity.minimize(obj_f, num_evals=100, **kwargs)
with open(c_pair+'_'+opt+'.json','w') as f:
f.write(json.dumps(ret))
results[(c_pair,opt)] = ret
elif(opt == 'profit'):
ret = optunity.maximize(obj_f, num_evals=100, **kwargs)
with open(c_pair+'_'+opt+'.json','w') as f:
f.write(json.dumps(ret))
results[(c_pair,opt)] = ret
elif(opt == 'q_mse'):
ret = optunity.minimize(obj_f, num_evals=30, **kwargs)
with open(c_pair+'_'+opt+'.json','w') as f:
f.write(json.dumps(ret))
results[(c_pair,opt)] = ret
elif(opt == 'q_profit'):
ret = optunity.maximize(obj_f, num_evals=30, **kwargs)
with open(c_pair+'_'+opt+'.json','w') as f:
f.write(json.dumps(ret))
results[(c_pair,opt)] = ret
return results
def lr_tuned(x_train, y_train, x_test, y_test):
@optunity.cross_validated(x=x_train, y=y_train, num_folds=3)
def inner_cv(x_train, y_train, x_test, y_test, regularization, step):
predict, _, _ = train_lr(x_train,
y_train,
regularization=regularization,
step=step)
yhat = predict(x_test)
return optunity.metrics.logloss(y_test, yhat)
pars, _, _ = optunity.minimize(inner_cv,
num_evals=50,
regularization=[0.001, 0.05],
step=[0.01, 0.2])
predict, w, b = train_lr(x_train, y_train, **pars)
yhat = predict(x_test)
loss = optunity.metrics.logloss(y_test, yhat)
brier = optunity.metrics.brier(y_test, yhat)
print('+ model: ' + str(b.get_value())[:5] + ' + ' +
str(w.get_value()[0])[:5] + ' * x1 + ' + str(w.get_value()[1])[:5] +
' * x2')
print('++ log loss in test fold: ' + str(loss))
print('++ Brier loss in test fold: ' + str(brier))
print('')
return loss, brier
def plot_randomsearch():
solver = optunity.available_solvers()[3]
pars, details, _ = optunity.minimize(f,
num_evals=100,
x=[-5, 5],
y=[-5, 5],
solver_name=solver)
def compute_mse_rbf_tuned(x_train, y_train, x_test, y_test):
"""Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""
# define objective function for tuning
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma, epsilon):
pipe = Pipeline([('scaler', preprocessing.StandardScaler()),
('svr', SVR(C=C, gamma=gamma, epsilon=epsilon))])
model = pipe.fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# optimize parameters
optimal_pars, _, _ = optunity.minimize(tune_cv,
150,
C=[1, 100],
gamma=[0, 50],
epsilon=[0, 2])
print("optimal hyperparameters: " + str(optimal_pars))
model = Pipeline([('scaler', preprocessing.StandardScaler()),
('svr', SVR(**optimal_pars))])
tuned_model = model.fit(x_train, y_train)
predictions = tuned_model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
def compute_mse_rbf_tuned(x_train, y_train, x_test, y_test):
"""Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""
global optimal_parameters, clf
# define objective function for tuning
@optunity.cross_validated(x=x_train,
y=y_train,
num_iter=2,
num_folds=2)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
# sample_weights = my_scaling_odr(y_train)
# sample_weights = [i / max(Events[-1]) for i in Events[-1]]
model = svm.SVR(C=C, gamma=gamma).fit(
x_train, y_train) #, sample_weight=sample_weights
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# optimize parameters
optimal_pars, _, _ = optunity.minimize(tune_cv,
200,
C=[1, 4000],
gamma=[0, 10],
pmap=optunity.pmap)
logging.info("Optimal hyperparameters: " + str(optimal_pars))
# sample_weights = my_scaling_odr(y_train)
tuned_model = svm.SVR(**optimal_pars).fit(x_train, y_train)
predictions = tuned_model.predict(x_test)
mse = optunity.metrics.mse(y_test, predictions)
logging.info('mse: ' + str(mse))
if mse < mse_old:
optimal_parameters = optimal_pars
clf = tuned_model
return mse
def optimize_objective(f):
logs = {}
solvers = optunity.available_solvers()
for solver in solvers:
pars, details, _ = optunity.minimize(f,
num_evals=100,
x=[-5, 5],
y=[-5, 5],
solver_name=solver)
logs[solver] = np.array(
[details.call_log['args']['x'], details.call_log['args']['y']])
colors = ['r', 'g', 'b', 'y', 'k', 'y', 'r', 'g']
markers = ['x', '+', 'o', 's', 'p', 'x', '+', 'o']
# compute contours of the objective function
delta = 0.025
x = np.arange(-5.0, 5.0, delta)
y = np.arange(-5.0, 5.0, delta)
X, Y = np.meshgrid(x, y)
Z = f(X, Y)
CS = plt.contour(X, Y, Z)
plt.clabel(CS, inline=1, fontsize=8, alpha=0.5)
for i, solver in enumerate(solvers):
plt.scatter(logs[solver][0, :],
logs[solver][1, :],
c=colors[i],
marker=markers[i],
alpha=0.80)
plt.xlim([-5, 5])
plt.ylim([-5, 5])
plt.axis('equal')
plt.legend(solvers)
plt.show()
def svr_tuned_predictions(x_train, y_train, x_test, y_test):
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
optimal_pars, _, _ = optunity.minimize(tune_cv, 200, C=[0, 20],
gamma=[0, 10], pmap=optunity.pmap)
tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
return tuned_model.predict(x_test).tolist()
def initialrdf(volmax, volmin, reagent, wellnum):
_, info_random, _ = optunity.minimize(f,
num_evals=wellnum,
x1=[volmin, volmax],
solver_name='random search')
##Create quasi-random data spread
reagentlist = info_random.call_log['args'][
'x1'] #Create reagent amounts (mmol), change x variable to change range, each generated from unique sobol index
reagentname = "Reagent%s (ul)" % reagent
rdf = pd.DataFrame({reagentname: reagentlist}).astype(int)
return (rdf)
def time_series():
# the order of examples is dependent on time, and must be preserved
# i.e. we can not randomly shuffle the data
x_train, y_train, x_test, y_test = dataset_split(d, split=0.8)
dictSize = 200
def f(gamma, forget, eta, nu):
gamma = np.exp(gamma)
return regression(x_train, y_train, x_test, y_test, gamma, forget, eta, nu, dictSize, RP=rij)
return optunity.minimize(f, num_evals=n_evals, gamma=[-10,0], \
forget=[0.98,1], eta=[0,0.5], nu=[0, 1], solver_name=solver )
def normal():
# we use a different optimizer. one which we can do cross-validation.
X, Y, _, _ = dataset_split(d, split=1)
dictSize = 200
@optunity.cross_validated(x=X, y=Y, num_folds=n_folds)
def f(x_train, y_train, x_test, y_test, gamma, forget, eta, nu):
gamma = np.exp(gamma)
return regression(x_train, y_train, x_test, y_test, gamma, forget, eta, nu, dictSize, RP=rij)
return optunity.minimize(f, num_evals=n_evals, gamma=[-10,0], \
forget=[0.98,1], eta=[0,0.5], nu=[0, 1], solver_name=solver )
def svr_tuned_predictions(x_train, y_train, x_test, y_test):
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
optimal_pars, _, _ = optunity.minimize(tune_cv,
200,
C=[0, 20],
gamma=[0, 10],
pmap=optunity.pmap)
tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
return tuned_model.predict(x_test).tolist()
def model_selection():
pmap8 = optunity.parallel.create_pmap(8)
hps, _, _ = optunity.minimize(svr_mse, num_evals=400,
solver_name='particle swarm',
logC=[5, 7],
logGamma=[-6.5, -5],
logEpsilon=[.7, 2],
# pmap=optunity.pmap)
pmap=pmap8)
print(hps)
optimal_model = sklearn.svm.SVR(C=10 ** hps['logC'],
gamma=10 ** hps['logGamma'],
epsilon=10 ** hps['logEpsilon']
).fit(features, target)
joblib.dump(optimal_model, finalModel)
def lr_tuned(x_train, y_train, x_test, y_test):
@optunity.cross_validated(x=x_train, y=y_train, num_folds=3)
def inner_cv(x_train, y_train, x_test, y_test, regularization, step):
predict, _, _ = train_lr(x_train, y_train,
regularization=regularization, step=step)
yhat = predict(x_test)
return optunity.metrics.logloss(y_test, yhat)
pars, _, _ = optunity.minimize(inner_cv, num_evals=50,
regularization=[0.001, 0.05],
step=[0.01, 0.2])
predict, w, b = train_lr(x_train, y_train, **pars)
yhat = predict(x_test)
loss = optunity.metrics.logloss(y_test, yhat)
brier = optunity.metrics.brier(y_test, yhat)
return loss, brier
def compute_mse_svr_rbf(x_train, y_train, x_test, y_test):
"""Find the optimal hyperparameters of svm;
Train a model using the optmal parametes
compute MSE
"""
# -------------------------------------------------------------------------------------------------------------
# Find optimal parameters
# ------------------------
@optunity.cross_validated(x=x_train, y=y_train, num_iter=num_iter_inner_cv,
num_folds=num_folds_inner_cv)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
model = SVR(C=C, gamma=gamma).fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# Optimise parameters
optimal_pars, _, _ = optunity.minimize(tune_cv, num_evals=num_evals_inner_cv, C=C_svr_rbf, gamma=gamma_svr_rbf)
print("THe optimal hyperparameters of SVR (kernel = rbf): " + str(optimal_pars))
# -----------------------
# Find optimal parameters
# -------------------------------------------------------------------------------------------------------------
# Train a model using the optimal parameters and the x_train and y_train
tuned_model = SVR(**optimal_pars).fit(x_train, y_train)
# Predict the testing data and training data
predictions_train = tuned_model.predict(x_train)
predictions_train = predictions_train.reshape(x_train.shape[0], order='C') # Make it one-D
predictions_test = tuned_model.predict(x_test)
predictions_test = predictions_test.reshape(x_test.shape[0], order='C')
# Errors
errors_train = errors_prediction(y_train, predictions_train)
errors_test = errors_prediction(y_test, predictions_test)
print('R^2_train: ', errors_train['r2_score'])
print('R^2_test:', errors_test['r2_score'])
# Save the parameters and errors
predictions_labels_each_fold.append({'predictions_train': predictions_train,
'labels_train': y_train,
'predictions_test': predictions_test,
'labels_test': y_test})
params_each_fold.append(optimal_pars)
errors_each_fold.append({'errors_train': errors_train, 'errors_test': errors_test})
tuned_models_each_fold.append(tuned_model)
return errors_test['mse']
def compute_mse_tuned(x_train, y_train, x_test, y_test):
# define objective function
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# optimize parameters
optimal_pars, _, _ = optunity.minimize(tune_cv, 200, C=[0, 10], gamma=[0, 10], pmap=optunity.pmap)
# if you are running this in IPython, optunity.pmap will not work
# more info at: https://github.com/claesenm/optunity/issues/8
# comment out the above line and replace by the one below:
# optimal_pars, _, _ = optunity.minimize(inner_cv, 150, C=[0, 10], gamma=[0, 0.1])
tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
predictions = tuned_model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
def compute_mse_rbf_tuned(x_train, y_train, x_test, y_test):
"""Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""
# define objective function for tuning
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
print("tune_cv model C="+str(C)+", gamma="+str(gamma))
model = SVR(C=C, gamma=gamma).fit(x_train, y_train)
print("tune_cv model fit")
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# optimize parameters
optimal_pars, _, _ = optunity.minimize(tune_cv, 150, C=[1, 100], gamma=[0, 50])
print("optimal hyperparameters: " + str(optimal_pars))
tuned_model = SVR(**optimal_pars).fit(x_train, y_train)
predictions = tuned_model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
def lr_tuned(x_train, y_train, x_test, y_test):
@optunity.cross_validated(x=x_train, y=y_train, num_folds=3)
def inner_cv(x_train, y_train, x_test, y_test, regularization, step):
predict, _, _ = train_lr(x_train,
y_train,
regularization=regularization,
step=step)
yhat = predict(x_test)
return optunity.metrics.logloss(y_test, yhat)
pars, _, _ = optunity.minimize(inner_cv,
num_evals=50,
regularization=[0.001, 0.05],
step=[0.01, 0.2])
predict, w, b = train_lr(x_train, y_train, **pars)
yhat = predict(x_test)
loss = optunity.metrics.logloss(y_test, yhat)
brier = optunity.metrics.brier(y_test, yhat)
return loss, brier
def lr_tuned(x_train, y_train, x_test, y_test):
@optunity.cross_validated(x=x_train, y=y_train, num_folds=3)
def inner_cv(x_train, y_train, x_test, y_test, regularization, step):
predict, _, _ = train_lr(x_train, y_train,
regularization=regularization, step=step)
yhat = predict(x_test)
return optunity.metrics.logloss(y_test, yhat)
pars, _, _ = optunity.minimize(inner_cv, num_evals=50,
regularization=[0.001, 0.05],
step=[0.01, 0.2])
predict, w, b = train_lr(x_train, y_train, **pars)
yhat = predict(x_test)
loss = optunity.metrics.logloss(y_test, yhat)
brier = optunity.metrics.brier(y_test, yhat)
print('+ model: ' + str(b.get_value())[:5] + ' + ' + str(w.get_value()[0])[:5] + ' * x1 + ' + str(w.get_value()[1])[:5] + ' * x2')
print('++ log loss in test fold: ' + str(loss))
print('++ Brier loss in test fold: ' + str(brier))
print('')
return loss, brier
def normal():
# we use a different optimizer. one which we can do cross-validation.
X, Y, _, _ = dataset_split(d, split=1)
dictSize = 200
@optunity.cross_validated(x=X, y=Y, num_folds=n_folds)
def f(x_train, y_train, x_test, y_test, gamma, forget, eta, nu):
gamma = np.exp(gamma)
return regression(x_train,
y_train,
x_test,
y_test,
gamma,
forget,
eta,
nu,
dictSize,
RP=rij)
return optunity.minimize(f, num_evals=n_evals, gamma=[-10,0], \
forget=[0.98,1], eta=[0,0.5], nu=[0, 1], solver_name=solver )
def time_series():
# the order of examples is dependent on time, and must be preserved
# i.e. we can not randomly shuffle the data
x_train, y_train, x_test, y_test = dataset_split(d, split=0.8)
dictSize = 200
def f(gamma, forget, eta, nu):
gamma = np.exp(gamma)
return regression(x_train,
y_train,
x_test,
y_test,
gamma,
forget,
eta,
nu,
dictSize,
RP=rij)
return optunity.minimize(f, num_evals=n_evals, gamma=[-10,0], \
forget=[0.98,1], eta=[0,0.5], nu=[0, 1], solver_name=solver )
def compute_mse_rbf_tuned(x_train, y_train, x_test, y_test):
"""Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""
# define objective function for tuning
@optunity.cross_validated(x=x_train, y=y_train,
num_folds=3) # num_iter=2,
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# optimize parameters
optimal_pars, _, _ = optunity.minimize(tune_cv,
150,
C=[55000, 65000],
gamma=[0, 1])
print("optimal hyperparameters: " + str(optimal_pars))
tuned_model = sklearn.svm.SVR(epsilon=0.01, **optimal_pars).fit(
x_train, y_train) # epsilon = 0.001,
predictions = tuned_model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
def compute_mse_tuned(x_train, y_train, x_test, y_test):
# define objective function
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# optimize parameters
optimal_pars, _, _ = optunity.minimize(tune_cv,
200,
C=[0, 10],
gamma=[0, 10],
pmap=optunity.pmap)
# if you are running this in IPython, optunity.pmap will not work
# more info at: https://github.com/claesenm/optunity/issues/8
# comment out the above line and replace by the one below:
# optimal_pars, _, _ = optunity.minimize(inner_cv, 150, C=[0, 10], gamma=[0, 0.1])
tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
predictions = tuned_model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
def compute_mse_poly_tuned(x_train, y_train, x_test, y_test):
"""Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""
#
# define objective function for tuning
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
def tune_cv(x_train, y_train, x_test, y_test, C, degree, coef0):
model = sklearn.svm.SVR(C=C, degree=degree, coef0=coef0,
kernel='poly').fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
#
# optimize parameters
optimal_pars, _, _ = optunity.minimize(tune_cv,
150,
C=[1000, 20000],
degree=[2, 5],
coef0=[0, 1])
print("optimal hyperparameters: " + str(optimal_pars))
#
tuned_model = sklearn.svm.SVR(kernel='poly',
**optimal_pars).fit(x_train, y_train)
predictions = tuned_model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# This file shows an example of how to retain all information during cross-validation per fold.
# For this we use the optunity.cross_validation.mean_and_list aggregator.
# For more context, cfr. https://github.com/claesenm/optunity/issues/40
import optunity
import optunity.cross_validation
x = list(range(5))
@optunity.cross_validated(x,
num_folds=5,
aggregator=optunity.cross_validation.mean_and_list)
def f(x_train, x_test, coeff):
return x_test[0] * coeff
# evaluate f
foo = f(0.0)
print(foo)
opt_coeff, info, _ = optunity.minimize(f, coeff=[0, 1], num_evals=10)
print(opt_coeff)
print("call log")
for args, val in zip(info.call_log['args']['coeff'], info.call_log['values']):
print(str(args) + '\t\t' + str(val))
return(y_pred)
def rf_pso(x_train, y_train, x_valid):
# Random forest using particle swarm opt on the hyperparameters
# pso
x_train = x_train.as_matrix()
y_train = y_train.as_matrix()
@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
def performance(x_train, y_train, x_test, y_test,n_estimators, max_features):
model = RandomForestRegressor(n_estimators=int(n_estimators),
max_features=max_features)
model.fit(x_train, y_train)
predictions = model.predict(x_test)
return(optunity.metrics.mse(y_test, predictions))
# hyperparameter search
optimal_pars, _, _ = optunity.minimize(performance, num_evals = 100, n_estimators=[10,800], max_features=[0.2,10])
# train
forest_reg = RandomForestRegressor(random_state=SEED, **optimal_pars)
reg.fit(x_train, y_train)
# validate
y_pred = reg.predict(x_valid)
# return predictive probabilities
return(y_pred)
def rf_gridsearch(x_train, y_train, x_valid):
# Random forest using grid search
# train
forest_reg = RandomForestRegressor(random_state=SEED)
param_grid = {'n_estimators': [3, 250, 500], 'max_features': [0.2, 0.4, 0.6, 0.8, 1]}
forest_grid = GridSearchCV(forest_reg, param_grid, cv=5, scoring='mean_absolute_error')
forest_grid.fit(x_train, y_train)
1]
while (np.sum(np.sum(Y[train, :], 0) < 5) > 0):
perm = np.random.permutation(X.shape[0])
train = perm[0:train_samp + 1]
test = perm[train_samp + 1:train_samp + test_samp + 1]
val = perm[train_samp + test_samp + 1:train_samp + test_samp +
val_samp + 1]
prt = False
opt_params = None
opt_params, _, _ = opt.minimize(run_LAmbDA,
solver_name='sobol',
gamma=[0.8, 1.2],
delta=[0.05, 0.95],
tau=[1.0, 2.0],
prc_cut=[20, 50],
bs_prc=[0.2, 0.6],
do_prc=[0.5, 1],
hidden_feats=[50, 150],
lambda1=[0, 5],
lambda2=[3, 5],
lambda3=[3, 5],
num_evals=10)
prt = True
train = perm[0:train_samp + test_samp + 1]
test = val
err = run_LAmbDA(opt_params['gamma'], opt_params['delta'],
opt_params['tau'], opt_params['prc_cut'],
opt_params['bs_prc'], opt_params['do_prc'],
opt_params['hidden_feats'], opt_params['lambda1'],
opt_params['lambda2'], opt_params['lambda3'])
tf.reset_default_graph()
def run_LAmbDA(DataPath,
LabelsPath,
CV_RDataPath,
OutputDir,
GeneOrderPath="",
NumGenes=0):
'''
run LAmbDA classifier
Wrapper script to run LAmbDA on a benchmark dataset with 5-fold cross validation,
outputs lists of true and predicted cell labels as csv files, as well as computation time.
Parameters
----------
DataPath : Data file path (.csv), cells-genes matrix with cell unique barcodes
as row names and gene names as column names.
LabelsPath : Cell population annotations file path (.csv).
CV_RDataPath : Cross validation RData file path (.RData), obtained from Cross_Validation.R function.
OutputDir : Output directory defining the path of the exported file.
GeneOrderPath : Gene order file path (.csv) obtained from feature selection,
defining the genes order for each cross validation fold, default is NULL.
NumGenes : Number of genes used in case of feature selection (integer), default is 0.
'''
# read the Rdata file
robjects.r['load'](CV_RDataPath)
nfolds = np.array(robjects.r['n_folds'], dtype='int')
tokeep = np.array(robjects.r['Cells_to_Keep'], dtype='bool')
col = np.array(robjects.r['col_Index'], dtype='int')
col = col - 1
test_ind = np.array(robjects.r['Test_Idx'])
train_ind = np.array(robjects.r['Train_Idx'])
# read the data
data = pd.read_csv(DataPath, index_col=0, sep=',')
labels = pd.read_csv(LabelsPath,
header=0,
index_col=None,
sep=',',
usecols=col)
labels = labels.iloc[tokeep]
data = data.iloc[tokeep]
# read the feature file
if (NumGenes > 0):
features = pd.read_csv(GeneOrderPath,
header=0,
index_col=None,
sep=',')
# folder with results
os.chdir(OutputDir)
tr_time = []
ts_time = []
truelab = np.zeros([len(labels), 1], dtype=int)
predlab = np.zeros([len(labels), 1], dtype=int)
for i in range(np.squeeze(nfolds)):
global X, Y, Gnp, Dnp, train, test, prt, cv
test_ind_i = np.array(test_ind[i], dtype='int') - 1
train_ind_i = np.array(train_ind[i], dtype='int') - 1
X = np.array(data)
if (NumGenes > 0):
X = np.log2(X / 10 + 1)
feat_to_use = features.iloc[0:NumGenes, i]
X = X[:, feat_to_use]
else:
X = np.log2(
np.transpose(select_feats(np.transpose(X), 0.5, 80)) / 10 + 1)
uniq = np.unique(labels)
Y = np.zeros([len(labels), len(uniq)], int)
for j in range(len(uniq)):
Y[np.where(labels == uniq[j])[0], j] = 1
Y = np.array(Y)
Gnp = np.zeros([len(uniq), len(uniq)], int)
np.fill_diagonal(Gnp, 1)
Gnp = np.array(Gnp)
Dnp = np.ones([len(uniq), 1], int)
Dnp = np.array(Dnp)
train_samp = int(np.floor(0.75 * len(train_ind_i)))
test_samp = len(train_ind_i) - train_samp
perm = np.random.permutation(len(train_ind_i))
train = perm[0:train_samp]
test = perm[train_samp:test_samp + 1]
while (np.sum(np.sum(Y[train, :], 0) < 5) > 0):
perm = np.random.permutation(X.shape[0])
train = perm[0:train_samp + 1]
test = perm[train_samp + 1:train_samp + test_samp + 1]
cv = i
optunity_it = 0
prt = False
opt_params = None
start = tm.time()
opt_params, _, _ = opt.minimize(run_LAmbDA2,
solver_name='sobol',
gamma=[0.8, 1.2],
delta=[0.05, 0.95],
tau=[10.0, 11.0],
prc_cut=[20, 50],
bs_prc=[0.2, 0.6],
num_trees=[10, 200],
max_nodes=[100, 1000],
num_evals=50)
tr_time.append(tm.time() - start)
print("Finished training!")
prt = True
train = train_ind_i
test = test_ind_i
start = tm.time()
err = run_LAmbDA2(opt_params['gamma'], opt_params['delta'],
opt_params['tau'], opt_params['prc_cut'],
opt_params['bs_prc'], opt_params['num_trees'],
opt_params['max_nodes'])
ts_time.append(tm.time() - start)
tf.reset_default_graph()
predfile = 'preds_cv' + str(cv) + '.mat'
truefile = 'truth_cv' + str(cv) + '.mat'
pred = sio.loadmat(predfile)
truth = sio.loadmat(truefile)
pred = pred['preds']
truth = truth['labels']
pred_ind = np.argmax(pred, axis=1)
truth_ind = np.argmax(truth, axis=1)
predlab[test_ind_i, 0] = pred_ind
truelab[test_ind_i, 0] = truth_ind
truelab = pd.DataFrame(truelab)
predlab = pd.DataFrame(predlab)
tr_time = pd.DataFrame(tr_time)
ts_time = pd.DataFrame(ts_time)
if (NumGenes == 0):
truelab.to_csv("LAmbDA_True_Labels.csv", index=False)
predlab.to_csv("LAmbDA_Pred_Labels.csv", index=False)
tr_time.to_csv("LAmbDA_Training_Time.csv", index=False)
ts_time.to_csv("LAmbDA_Testing_Time.csv", index=False)
else:
truelab.to_csv("LAmbDA_" + str(NumGenes) + "_True_Labels.csv",
index=False)
predlab.to_csv("LAmbDA_" + str(NumGenes) + "_Pred_Labels.csv",
index=False)
tr_time.to_csv("LAmbDA_" + str(NumGenes) + "_Training_Time.csv",
index=False)
ts_time.to_csv("LAmbDA_" + str(NumGenes) + "_Testing_Time.csv",
index=False)
def search_param(self, database, dataprocess=None, path_filename=("", ""),
save=False, cv="ts", of="rmse", f=None, eval=50):
"""
Search best hyperparameters for classifier/regressor based on
optunity algorithms.
Arguments:
database (numpy.ndarray): a matrix containing all patterns
that will be used for training/testing at some
cross-validation method.
dataprocess (DataProcess): an object that will pre-process
database before training. Defaults to None.
path_filename (tuple): *TODO*.
save (bool): *TODO*.
cv (str): Cross-validation method. Defaults to "ts".
of (str): Objective function to be minimized at
optunity.minimize. Defaults to "rmse".
f (list of str): a list of functions to be used by the
search. Defaults to None, this set all available
functions.
eval (int): Number of steps (evaluations) to optunity algorithm.
Each set of hyperparameters will perform a cross-validation
method chosen by param cv.
Available *cv* methods:
- "ts" :func:`mltools.time_series_cross_validation()`
Perform a time-series cross-validation suggested by Hydman.
- "kfold" :func:`mltools.kfold_cross_validation()`
Perform a k-fold cross-validation.
Available *of* function:
- "accuracy", "rmse", "mape", "me".
See Also:
http://optunity.readthedocs.org/en/latest/user/index.html
"""
if f is None:
search_functions = self.available_functions
elif type(f) is list:
search_functions = f
else:
raise Exception("Invalid format for argument 'f'.")
print(self.regressor_name)
print("##### Start search #####")
config = configparser.ConfigParser()
if sys.version_info < (3, 0):
config.readfp(open(_ELMR_CONFIG))
else:
config.read_file(open(_ELMR_CONFIG))
best_function_error = 99999.9
temp_error = best_function_error
best_param_function = ""
best_param_c = 0
best_param_l = 0
for function in search_functions:
if sys.version_info < (3, 0):
elmr_c_range = ast.literal_eval(config.get("DEFAULT",
"elmr_c_range"))
neurons = config.getint("DEFAULT", "elmr_neurons")
else:
function_config = config["DEFAULT"]
elmr_c_range = ast.literal_eval(function_config["elmr_c_range"])
neurons = ast.literal_eval(function_config["elmr_neurons"])
param_ranges = [[elmr_c_range[0][0], elmr_c_range[0][1]]]
def wrapper_opt(param_c):
"""
Wrapper for optunity.
"""
if cv == "ts":
cv_tr_error, cv_te_error = \
time_series_cross_validation(self, database,
params=[function,
2 ** param_c,
neurons,
False],
number_folds=10,
dataprocess=dataprocess)
elif cv == "kfold":
cv_tr_error, cv_te_error = \
kfold_cross_validation(self, database,
params=[function,
2 ** param_c,
neurons,
False],
number_folds=10,
dataprocess=dataprocess)
else:
raise Exception("Invalid type of cross-validation.")
if of == "accuracy":
util = 1 / cv_te_error.get_accuracy()
else:
util = cv_te_error.get(of)
# print("c:", param_c, "util: ", util)
return util
optimal_pars, details, _ = \
optunity.minimize(wrapper_opt,
solver_name="cma-es",
num_evals=eval,
param_c=param_ranges[0])
# Save best function result
if details[0] < temp_error:
temp_error = details[0]
if of == "accuracy":
best_function_error = 1 / temp_error
else:
best_function_error = temp_error
best_param_function = function
best_param_c = optimal_pars["param_c"]
best_param_l = neurons
if of == "accuracy":
print("Function: ", function,
" best cv value: ", 1/details[0])
else:
print("Function: ", function,
" best cv value: ", details[0])
# MLTools Attribute
self.cv_best_rmse = best_function_error
# elmr Attribute
self.param_function = best_param_function
self.param_c = best_param_c
self.param_l = best_param_l
print("##### Search complete #####")
self.print_parameters()
return None
def main():
parser = argparse.ArgumentParsers(
description="Run grid optimization on a single subject")
parser.add_argument('msh',
type=str,
help="Subject Gmsh .msh realistic head model")
parser.add_argument('weights',
type=str,
help=".npy binary containing a weight for each "
"tetrahedron")
parser.add_argument('centroid',
type=str,
help="Coordinates in T1w space for a centroid "
"to the weight function to optimize over")
parser.add_argument('coil', type=str, help="Path to SimNIBS coil file")
parser.add_argument('output_file',
type=str,
help="Output file storing optimal coordinates")
parser.add_argument('output_file',
type=str,
help="Output file storing optimal coordinates")
parser.add_argument('--history',
type=str,
help="Output file to store history of scores"
" into for convergence/visualization")
parser.add_argument('--workdir',
type=str,
help="Working directory to run simulations in")
parser.add_argument('--ncpus',
type=int,
help="Number of threads to use for each batch "
"of simulations. Default = 8")
parser.add_argument('--batchsize',
type=int,
help="Number of simulations to run simultaneously, "
"will default to half the number of cpus if not "
"specified.")
parser.add_argument('--solver',
type=int,
help="Optunity solver to use, "
"defaults to particle swarm",
choices=optunity.available_solvers())
args = parser.parse()
msh = args.msh
wf = np.load(args.weights)
centroid = np.genfromtxt(args.centroid)
coil = args.coil
ncpus = args.ncpus or 8
batch_size = args.batchsize or (ncpus // 2 - 1)
history = args.history
workdir = args.workdir or "/tmp/"
output_file = args.output_file
solver = args.solver or "particle swarm"
# Construct objective function object
f = FieldFunc(mesh_file=msh,
initial_centroid=centroid,
tet_weights=wf,
coil=coil,
field_dir=workdir,
cpus=ncpus)
# Set up optunity optimization
# Can we feed a list of inputs here?
pars, details, _ = optunity.minimize(f.evaluate,
num_evals=100,
x=[f.bounds[0, 0], f.bounds[0, 1]],
y=[f.bounds[1, 0], f.bounds[1, 1]],
theta=[0, 180],
solver_name=solver)
def modelling_svr_rbf(C_svr_rbf,
gamma_svr_rbf,
wavelengths_range,
input_type,
num_folds_outer_cv,
num_iter_inner_cv,
num_folds_inner_cv,
num_evals_inner_cv,
samples,
wavelengths,
labels,
flag_save,
flag_fig):
""" Model a svr with rbf kernel."""
start = datetime.datetime.now()
print('')
print('svr (kernel = rbf)')
print('The range of C is: ', C_svr_rbf)
print('The range of gamma is: ', gamma_svr_rbf)
print('')
# For records
date_time = datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S')
model_name = 'svr_rbf'
save_record_name = 'record' + '_' + wavelengths_range + '_' + input_type + '_' + model_name + '.sav'
save_model_name = 'model' + '_' + wavelengths_range + '_' + input_type + '_' + model_name + '.sav'
# //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
# CV
# ///
print('Conducting cross-validation')
# For record
params_each_fold = []
errors_each_fold = []
predictions_labels_each_fold = []
tuned_models_each_fold = []
# ==================================================================================================================
# Thf function for outer_cv
# ==========================
def compute_mse_svr_rbf(x_train, y_train, x_test, y_test):
"""Find the optimal hyperparameters of svm;
Train a model using the optmal parametes
compute MSE
"""
# -------------------------------------------------------------------------------------------------------------
# Find optimal parameters
# ------------------------
@optunity.cross_validated(x=x_train, y=y_train, num_iter=num_iter_inner_cv,
num_folds=num_folds_inner_cv)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
model = SVR(C=C, gamma=gamma).fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# Optimise parameters
optimal_pars, _, _ = optunity.minimize(tune_cv, num_evals=num_evals_inner_cv, C=C_svr_rbf, gamma=gamma_svr_rbf)
print("THe optimal hyperparameters of SVR (kernel = rbf): " + str(optimal_pars))
# -----------------------
# Find optimal parameters
# -------------------------------------------------------------------------------------------------------------
# Train a model using the optimal parameters and the x_train and y_train
tuned_model = SVR(**optimal_pars).fit(x_train, y_train)
# Predict the testing data and training data
predictions_train = tuned_model.predict(x_train)
predictions_train = predictions_train.reshape(x_train.shape[0], order='C') # Make it one-D
predictions_test = tuned_model.predict(x_test)
predictions_test = predictions_test.reshape(x_test.shape[0], order='C')
# Errors
errors_train = errors_prediction(y_train, predictions_train)
errors_test = errors_prediction(y_test, predictions_test)
print('R^2_train: ', errors_train['r2_score'])
print('R^2_test:', errors_test['r2_score'])
# Save the parameters and errors
predictions_labels_each_fold.append({'predictions_train': predictions_train,
'labels_train': y_train,
'predictions_test': predictions_test,
'labels_test': y_test})
params_each_fold.append(optimal_pars)
errors_each_fold.append({'errors_train': errors_train, 'errors_test': errors_test})
tuned_models_each_fold.append(tuned_model)
return errors_test['mse']
# =========================
# The function for outer cv
# ==================================================================================================================
# The fellow is the same as:
# @optunity.cross_validated(x=samples, y=labels, num_folds=num_folds_outer_cv)
# def compute_mse_svr_rbf:
# ...
#
# compute_mse_svr_rbf()
outer_cv = optunity.cross_validated(x=samples, y=labels, num_folds=num_folds_outer_cv) # function decoter
compute_mse_svr_rbf = outer_cv(compute_mse_svr_rbf) # Decorate computer_mse_svr_rbf
compute_mse_svr_rbf()
print('The cross-validation has been done!', datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S'))
stop = datetime.datetime.now()
print('Total time used ', stop - start)
# ///
# CV
# //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
# Record the results
ave_errors = errors_average(errors_each_fold)
record_svr_rbf = {'model_name': save_model_name,
'date_time': date_time,
'C_range': C_svr_rbf,
'gamma_range': gamma_svr_rbf,
'num_folds_outer_cv': num_folds_outer_cv,
'num_iter_inner_cv': num_iter_inner_cv,
'num_folds_inner_cv': num_folds_inner_cv,
'num_evals_inner_cv': num_evals_inner_cv,
'tuned_models_each_fold': tuned_models_each_fold,
'predictions_labels_each_fold': predictions_labels_each_fold,
'optimal_parameters_each_fold': params_each_fold,
'errors_each_fold': errors_each_fold,
'average_errors': ave_errors,
'wavelengths': wavelengths
}
# Print average of cv
print_ave_errors_cv(ave_errors)
if flag_fig:
# Plot a record in one (random selected) of the cv
plot_regression_result(predictions_labels_each_fold[0]['labels_train'],
predictions_labels_each_fold[0]['predictions_train'])
plot_regression_result(predictions_labels_each_fold[0]['labels_test'],
predictions_labels_each_fold[0]['predictions_test'])
# //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
# Train a model using all of the data
# ////////////////////////////////////
# ==================================================================================================================
# Find the optimal parameters
# ============================
print('Training a SVR (kernel = rbf) instance.')
@optunity.cross_validated(x=samples, y=labels, num_iter=num_iter_inner_cv,
num_folds=num_folds_inner_cv)
def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
model = SVR(C=C, gamma=gamma).fit(x_train, y_train)
predictions = model.predict(x_test)
return optunity.metrics.mse(y_test, predictions)
# Optimise parameters
optimal_pars, _, _ = optunity.minimize(tune_cv, num_evals=num_evals_inner_cv, C=C_svr_rbf, gamma=gamma_svr_rbf)
# ============================
# Find the optimal parameters
# ==================================================================================================================
# Train a model using all of the data
tuned_model_finial = SVR(**optimal_pars).fit(samples, labels)
# ///////////////////////////////////
# Train a model using all of the data
# //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
# Save the model
if flag_save:
joblib.dump(record_svr_rbf, save_record_name)
joblib.dump(tuned_model_finial, save_model_name)
print('The tuned_model_finial and the record has been saved!')
return record_svr_rbf, tuned_model_finial
# SVM regression with rbf kernel
################################
# SVM regression with rbf kernel
########################################################################################################################
return optunity.metrics.roc_auc(y_test, predictions, positive=True)
performance(algorithm='k-nn', n_neighbors=3)
optimal_configuration, info, _ = optunity.maximize_structured(performance,
search_space=search,
num_evals=300)
print(optimal_configuration)
print(info.optimum)
solution = dict([(k, v) for k, v in optimal_configuration.items() if v is not None])
print('Solution\n========')
print("\n".join(map(lambda x: "%s \t %s" % (x[0], str(x[1])), solution.items())))
#basic optim
def create_objective_function():
xoff = random.random()
yoff = random.random()
def f(x, y):
return (x - xoff)**2 + (y - yoff)**2
return f
solvers = optunity.available_solvers()
print('Available solvers: ' + ', '.join(solvers))
f = create_objective_function()
logs = {}
for solver in solvers:
pars, details, _ = optunity.minimize(f, num_evals=100, x=[-5, 5], y=[-5, 5],solver_name=solver)
logs[solver] = np.array([details.call_log['args']['x'],
details.call_log['args']['y']])
# This file shows an example of how to retain all information during cross-validation per fold.
# For this we use the optunity.cross_validation.mean_and_list aggregator.
# For more context, cfr. https://github.com/claesenm/optunity/issues/40
import optunity
import optunity.cross_validation
x = list(range(5))
@optunity.cross_validated(x, num_folds=5,
aggregator=optunity.cross_validation.mean_and_list)
def f(x_train, x_test, coeff):
return x_test[0] * coeff
# evaluate f
foo = f(0.0)
print(foo)
opt_coeff, info, _ = optunity.minimize(f, coeff=[0, 1], num_evals=10)
print(opt_coeff)
print("call log")
for args, val in zip(info.call_log['args']['coeff'], info.call_log['values']):
print(str(args) + '\t\t' + str(val))
def fit(self,
obj_fun=None,
param_range=None,
num_evals=1000,
solver_name="sobol",
maximize=True,
num_jobs=1,
engine="optunity",
random_state=1,
init_points=20,
n_iter=50,
acq='ucb',
kappa=2.576,
xi=0.0):
if engine == "optunity":
if obj_fun:
self.obj_fun = obj_fun
if param_range:
self.param_range = param_range
if num_jobs == 1:
if maximize:
self.optimal_params, self.info, _ = op.maximize(
self.obj_fun,
num_evals=num_evals,
solver_name=solver_name,
**self.param_range) # default: 'particle swarm'
else:
self.optimal_params, self.info, _ = op.minimize(
self.obj_fun,
num_evals=num_evals,
solver_name=solver_name,
**self.param_range) # de
# params
# print(self.optimal_params)
# optimum and states
print(self.info.optimum, self.info.stats)
elif num_jobs > 1:
if maximize:
self.optimal_params, self.info, _ = op.maximize(
self.obj_fun,
num_evals=num_evals,
solver_name=solver_name,
pmap=op.parallel.create_pmap(num_jobs),
**self.param_range) # default: 'particle swarm'
else:
self.optimal_params, self.info, _ = op.minimize(
self.obj_fun,
num_evals=num_evals,
solver_name=solver_name,
pmap=op.parallel.create_pmap(num_jobs),
**self.param_range) # de
# params
# print(self.optimal_params)
# optimum and states
print(self.info.optimum, self.info.stats)
elif engine == "bayes_opt":
if obj_fun:
self.obj_fun = obj_fun
if param_range:
self.param_range = param_range
param_range_t = {}
for key in self.param_range:
param_range_t[key] = tuple(self.param_range[key])
optimizer = BayesianOptimization(
f=self.obj_fun,
pbounds=param_range_t,
random_state=random_state,
)
optimizer.maximize(
init_points=init_points,
n_iter=n_iter,
acq=acq,
kappa=kappa,
xi=xi,
)
self.optimal_params = optimizer.max['params']
val = perm[train_samp + test_samp + 1:train_samp + test_samp + val_samp +
1]
while (np.sum(np.sum(Y[train, :], 0) < 5) > 0):
perm = np.random.permutation(X.shape[0])
train = perm[0:train_samp + 1]
test = perm[train_samp + 1:train_samp + test_samp + 1]
val = perm[train_samp + test_samp + 1:train_samp + test_samp +
val_samp + 1]
optunity_it = 0
prt = False
opt_params = None
opt_params, _, _ = opt.minimize(run_LAmbDA,
solver_name='sobol',
gamma=[0.8, 1.2],
delta=[0.05, 0.95],
tau=[10.0, 11.0],
prc_cut=[20, 50],
bs_prc=[0.2, 0.6],
num_trees=[10, 200],
max_nodes=[100, 1000],
num_evals=50)
print(opt_params)
prt = True
train = perm[0:train_samp + test_samp + 1]
test = val
err = run_LAmbDA(opt_params['gamma'], opt_params['delta'],
opt_params['tau'], opt_params['prc_cut'],
opt_params['bs_prc'], opt_params['num_trees'],
opt_params['max_nodes'])
tf.reset_default_graph()
print('Cross validation step: ' + str(cv))
print(opt_params)
def search_param(self, database, dataprocess=None, path_filename=("", ""),
save=False, cv="ts", of="rmse", kf=None, eval=50):
"""
Search best hyperparameters for classifier/regressor based on
optunity algorithms.
Arguments:
database (numpy.ndarray): a matrix containing all patterns
that will be used for training/testing at some
cross-validation method.
dataprocess (DataProcess): an object that will pre-process
database before training. Defaults to None.
path_filename (tuple): *TODO*.
save (bool): *TODO*.
cv (str): Cross-validation method. Defaults to "ts".
of (str): Objective function to be minimized at
optunity.minimize. Defaults to "rmse".
kf (list of str): a list of kernel functions to be used by
the search. Defaults to None, this set all available
functions.
eval (int): Number of steps (evaluations) to optunity algorithm.
Each set of hyperparameters will perform a cross-validation
method chosen by param cv.
Available *cv* methods:
- "ts" :func:`mltools.time_series_cross_validation()`
Perform a time-series cross-validation suggested by Hydman.
- "kfold" :func:`mltools.kfold_cross_validation()`
Perform a k-fold cross-validation.
Available *of* function:
- "accuracy", "rmse", "mape", "me".
See Also:
http://optunity.readthedocs.org/en/latest/user/index.html
"""
if kf is None:
search_kernel_functions = self.available_kernel_functions
elif type(kf) is list:
search_kernel_functions = kf
else:
raise Exception("Invalid format for argument 'kf'.")
print(self.regressor_name)
print("##### Start search #####")
config = configparser.ConfigParser()
if sys.version_info < (3, 0):
config.readfp(open(_ELMK_CONFIG))
else:
config.read_file(open(_ELMK_CONFIG))
best_function_error = 99999.9
temp_error = best_function_error
best_param_c = 0
best_param_kernel_function = ""
best_param_kernel_param = []
for kernel_function in search_kernel_functions:
if sys.version_info < (3, 0):
elmk_c_range = ast.literal_eval(config.get("DEFAULT",
"elmk_c_range"))
n_parameters = config.getint(kernel_function, "kernel_n_param")
kernel_p_range = \
ast.literal_eval(config.get(kernel_function,
"kernel_params_range"))
else:
kernel_config = config[kernel_function]
elmk_c_range = ast.literal_eval(kernel_config["elmk_c_range"])
n_parameters = int(kernel_config["kernel_n_param"])
kernel_p_range = \
ast.literal_eval(kernel_config["kernel_params_range"])
param_ranges = [[elmk_c_range[0][0], elmk_c_range[0][1]]]
for param in range(n_parameters):
param_ranges.append([kernel_p_range[param][0],
kernel_p_range[param][1]])
def wrapper_0param(param_c):
"""
Wrapper for objective function.
"""
if cv == "ts":
cv_tr_error, cv_te_error = \
time_series_cross_validation(self, database,
[kernel_function,
param_c,
list([])],
number_folds=10,
dataprocess=dataprocess)
elif cv == "kfold":
cv_tr_error, cv_te_error = \
kfold_cross_validation(self, database,
[kernel_function,
param_c,
list([])],
number_folds=10,
dataprocess=dataprocess)
else:
raise Exception("Invalid type of cross-validation.")
if of == "accuracy":
util = 1 / cv_te_error.get_accuracy()
else:
util = cv_te_error.get(of)
# print("c:", param_c, "util: ", util)
return util
def wrapper_1param(param_c, param_kernel):
"""
Wrapper for optunity.
"""
if cv == "ts":
cv_tr_error, cv_te_error = \
time_series_cross_validation(self, database,
[kernel_function,
param_c,
list([param_kernel])],
number_folds=10,
dataprocess=dataprocess)
elif cv == "kfold":
cv_tr_error, cv_te_error = \
kfold_cross_validation(self, database,
[kernel_function,
param_c,
list([param_kernel])],
number_folds=10,
dataprocess=dataprocess)
else:
raise Exception("Invalid type of cross-validation.")
if of == "accuracy":
util = 1 / cv_te_error.get_accuracy()
else:
util = cv_te_error.get(of)
# print("c:", param_c, " gamma:", param_kernel, "util: ", util)
return util
def wrapper_2param(param_c, param_kernel1, param_kernel2):
"""
Wrapper for optunity.
"""
if cv == "ts":
cv_tr_error, cv_te_error = \
time_series_cross_validation(self, database,
[kernel_function,
param_c,
list([param_kernel1,
param_kernel2])],
number_folds=10,
dataprocess=dataprocess)
elif cv == "kfold":
cv_tr_error, cv_te_error = \
kfold_cross_validation(self, database,
[kernel_function,
param_c,
list([param_kernel1,
param_kernel2])],
number_folds=10,
dataprocess=dataprocess)
else:
raise Exception("Invalid type of cross-validation.")
if of == "accuracy":
util = 1 / cv_te_error.get_accuracy()
else:
util = cv_te_error.get(of)
# print("c:", param_c, " param1:", param_kernel1,
# " param2:", param_kernel2, "util: ", util)
return util
if kernel_function == "linear":
optimal_parameters, details, _ = \
optunity.minimize(wrapper_0param,
solver_name="cma-es",
num_evals=eval,
param_c=param_ranges[0])
elif kernel_function == "rbf":
optimal_parameters, details, _ = \
optunity.minimize(wrapper_1param,
solver_name="cma-es",
num_evals=eval,
param_c=param_ranges[0],
param_kernel=param_ranges[1])
elif kernel_function == "poly":
optimal_parameters, details, _ = \
optunity.minimize(wrapper_2param,
solver_name="cma-es",
num_evals=eval,
param_c=param_ranges[0],
param_kernel1=param_ranges[1],
param_kernel2=param_ranges[2])
else:
raise Exception("Invalid kernel function.")
# Save best kernel result
if details[0] < temp_error:
temp_error = details[0]
if of == "accuracy":
best_function_error = 1 / temp_error
else:
best_function_error = temp_error
best_param_kernel_function = kernel_function
best_param_c = optimal_parameters["param_c"]
if best_param_kernel_function == "linear":
best_param_kernel_param = []
elif best_param_kernel_function == "rbf":
best_param_kernel_param = [optimal_parameters["param_kernel"]]
elif best_param_kernel_function == "poly":
best_param_kernel_param = \
[optimal_parameters["param_kernel1"],
optimal_parameters["param_kernel2"]]
else:
raise Exception("Invalid kernel function.")
# print("best: ", best_param_kernel_function,
# best_function_error, best_param_c, best_param_kernel_param)
if of == "accuracy":
print("Kernel function: ", kernel_function,
" best cv value: ", 1/details[0])
else:
print("Kernel function: ", kernel_function,
" best cv value: ", details[0])
# MLTools attribute
self.cv_best_rmse = best_function_error
# ELM attribute
self.param_c = best_param_c
self.param_kernel_function = best_param_kernel_function
self.param_kernel_params = best_param_kernel_param
print("##### Search complete #####")
self.print_parameters()
return None
def search_param(self,
database,
dataprocess=None,
path_filename=("", ""),
save=False,
cv="ts",
of="rmse",
f=None,
eval=50):
"""
Search best hyperparameters for classifier/regressor based on
optunity algorithms.
Arguments:
database (numpy.ndarray): a matrix containing all patterns
that will be used for training/testing at some
cross-validation method.
dataprocess (DataProcess): an object that will pre-process
database before training. Defaults to None.
path_filename (tuple): *TODO*.
save (bool): *TODO*.
cv (str): Cross-validation method. Defaults to "ts".
of (str): Objective function to be minimized at
optunity.minimize. Defaults to "rmse".
f (list of str): a list of functions to be used by the
search. Defaults to None, this set all available
functions.
eval (int): Number of steps (evaluations) to optunity algorithm.
Each set of hyperparameters will perform a cross-validation
method chosen by param cv.
Available *cv* methods:
- "ts" :func:`mltools.time_series_cross_validation()`
Perform a time-series cross-validation suggested by Hydman.
- "kfold" :func:`mltools.kfold_cross_validation()`
Perform a k-fold cross-validation.
Available *of* function:
- "accuracy", "rmse", "mape", "me".
See Also:
http://optunity.readthedocs.org/en/latest/user/index.html
"""
if f is None:
search_functions = self.available_functions
elif type(f) is list:
search_functions = f
else:
raise Exception("Invalid format for argument 'f'.")
print(self.regressor_name)
print("##### Start search #####")
config = configparser.ConfigParser()
if sys.version_info < (3, 0):
config.readfp(open(_ELMR_CONFIG))
else:
config.read_file(open(_ELMR_CONFIG))
best_function_error = 99999.9
temp_error = best_function_error
best_param_function = ""
best_param_c = 0
best_param_l = 0
for function in search_functions:
if sys.version_info < (3, 0):
elmr_c_range = ast.literal_eval(
config.get("DEFAULT", "elmr_c_range"))
neurons = config.getint("DEFAULT", "elmr_neurons")
else:
function_config = config["DEFAULT"]
elmr_c_range = ast.literal_eval(
function_config["elmr_c_range"])
neurons = ast.literal_eval(function_config["elmr_neurons"])
param_ranges = [[elmr_c_range[0][0], elmr_c_range[0][1]]]
def wrapper_opt(param_c):
"""
Wrapper for optunity.
"""
if cv == "ts":
cv_tr_error, cv_te_error = \
time_series_cross_validation(self, database,
params=[function,
2 ** param_c,
neurons,
False],
number_folds=10,
dataprocess=dataprocess)
elif cv == "kfold":
cv_tr_error, cv_te_error = \
kfold_cross_validation(self, database,
params=[function,
2 ** param_c,
neurons,
False],
number_folds=10,
dataprocess=dataprocess)
else:
raise Exception("Invalid type of cross-validation.")
if of == "accuracy":
util = 1 / cv_te_error.get_accuracy()
else:
util = cv_te_error.get(of)
# print("c:", param_c, "util: ", util)
return util
optimal_pars, details, _ = \
optunity.minimize(wrapper_opt,
solver_name="cma-es",
num_evals=eval,
param_c=param_ranges[0])
# Save best function result
if details[0] < temp_error:
temp_error = details[0]
if of == "accuracy":
best_function_error = 1 / temp_error
else:
best_function_error = temp_error
best_param_function = function
best_param_c = optimal_pars["param_c"]
best_param_l = neurons
if of == "accuracy":
print("Function: ", function, " best cv value: ",
1 / details[0])
else:
print("Function: ", function, " best cv value: ", details[0])
# MLTools Attribute
self.cv_best_rmse = best_function_error
# elmr Attribute
self.param_function = best_param_function
self.param_c = best_param_c
self.param_l = best_param_l
print("##### Search complete #####")
self.print_parameters()
return None
print(info.optimum)
solution = dict([(k, v) for k, v in optimal_configuration.items()
if v is not None])
print('Solution\n========')
print("\n".join(map(lambda x: "%s \t %s" % (x[0], str(x[1])),
solution.items())))
#basic optim
def create_objective_function():
xoff = random.random()
yoff = random.random()
def f(x, y):
return (x - xoff)**2 + (y - yoff)**2
return f
solvers = optunity.available_solvers()
print('Available solvers: ' + ', '.join(solvers))
f = create_objective_function()
logs = {}
for solver in solvers:
pars, details, _ = optunity.minimize(f,
num_evals=100,
x=[-5, 5],
y=[-5, 5],
solver_name=solver)
logs[solver] = np.array(
[details.call_log['args']['x'], details.call_log['args']['y']])
crps_train_min = crps_train
print('Saving weights...')
# save weights so they can be loaded later
model_systole.save_weights('weights_systole.hdf5', overwrite=True)
model_diastole.save_weights('weights_diastole.hdf5', overwrite=True)
# for best (lowest) val losses, save weights
if val_loss_systole < min_val_loss_systole:
min_val_loss_systole = val_loss_systole
model_systole.save_weights('weights_systole_best.hdf5', overwrite=True)
if val_loss_diastole < min_val_loss_diastole:
min_val_loss_diastole = val_loss_diastole
model_diastole.save_weights('weights_diastole_best.hdf5', overwrite=True)
# save best (lowest) val losses in file (to be later used for generating submission)
with open('val_loss.txt', mode='w+') as f:
f.write(str(min_val_loss_systole))
f.write('\n')
f.write(str(min_val_loss_diastole))
print('best case accuracy train: {0}, test: {1}'.format(crps_train_min, crps_test_min))
return crps_test_min
optimal_parameters, details, _ = optunity.minimize(train_accuracy, num_evals=20,
drop_out_1=[0.001, 1.0], drop_out_2=[0.001, 1.0], drop_out_3=[0.001, 1.0], drop_out_4=[0.001, 1.0])
print ('found optimal parameters: {0}'.format(optimal_parameters))
np.savetxt("best_acc_hyper_param.txt", optimal_parameters , delimiter=",")
#print(train_accuracy(0.25, 0.25, 0.25, 0.5))