def main():
    c_pairs = [f_1,f_2,f_3]
    kind = ['q_mse']
    options = []
    results = {}
    for c_pair in c_pairs:
        for opt in kind:
            options.append((c_pair,opt))
    for c_pair,opt in options:
        print(c_pair,opt)
        def obj_f(lag,drmse,krmse,window_size):
            f_lag = int(lag)
            f_drmse = np.exp(drmse)
            f_krmse = np.exp(krmse)
            f_window = int(window_size)
            obj_f.m += 1
            print(obj_f.m)
            if (opt == 'mse'):
                return trainMSE(c_pair+'.csv',f_lag,f_drmse,f_krmse,f_window)
            elif (opt == 'profit'):
                return trainProfit(c_pair+'.csv',f_lag,f_drmse,f_krmse,f_window)
            elif (opt == 'q_mse'):
                return trainMSEPredictLogRet(c_pair+'.csv',f_lag,f_drmse,f_krmse,f_window)
            elif (opt == 'q_profit'):
                return trainProfitPredictLogRet(c_pair+'.csv',f_lag,f_drmse,f_krmse,f_window)
        
        obj_f.m = 0
        kwargs = {'lag':[5,20],
                  'drmse':[np.log(0.001),np.log(0.01)], 
                  'krmse':[np.log(0.001),np.log(0.01)],
                  'window_size':[10,300]}
        
        ret = None
        if (opt == 'mse'):
            ret = optunity.minimize(obj_f, num_evals=100, **kwargs)
            with open(c_pair+'_'+opt+'.json','w') as f:
                f.write(json.dumps(ret))
            results[(c_pair,opt)] = ret
        elif(opt == 'profit'):
            ret = optunity.maximize(obj_f, num_evals=100, **kwargs)
            with open(c_pair+'_'+opt+'.json','w') as f:
                f.write(json.dumps(ret))
            results[(c_pair,opt)] = ret
        elif(opt == 'q_mse'):
            ret = optunity.minimize(obj_f, num_evals=30, **kwargs)
            with open(c_pair+'_'+opt+'.json','w') as f:
                f.write(json.dumps(ret))
            results[(c_pair,opt)] = ret
        elif(opt == 'q_profit'):
            ret = optunity.maximize(obj_f, num_evals=30, **kwargs)
            with open(c_pair+'_'+opt+'.json','w') as f:
                f.write(json.dumps(ret))
            results[(c_pair,opt)] = ret
    return results
def lr_tuned(x_train, y_train, x_test, y_test):
    @optunity.cross_validated(x=x_train, y=y_train, num_folds=3)
    def inner_cv(x_train, y_train, x_test, y_test, regularization, step):
        predict, _, _ = train_lr(x_train,
                                 y_train,
                                 regularization=regularization,
                                 step=step)
        yhat = predict(x_test)
        return optunity.metrics.logloss(y_test, yhat)

    pars, _, _ = optunity.minimize(inner_cv,
                                   num_evals=50,
                                   regularization=[0.001, 0.05],
                                   step=[0.01, 0.2])
    predict, w, b = train_lr(x_train, y_train, **pars)
    yhat = predict(x_test)
    loss = optunity.metrics.logloss(y_test, yhat)
    brier = optunity.metrics.brier(y_test, yhat)
    print('+ model: ' + str(b.get_value())[:5] + ' + ' +
          str(w.get_value()[0])[:5] + ' * x1 + ' + str(w.get_value()[1])[:5] +
          ' * x2')
    print('++ log loss in test fold: ' + str(loss))
    print('++ Brier loss in test fold: ' + str(brier))
    print('')
    return loss, brier
def plot_randomsearch():
    solver = optunity.available_solvers()[3]
    pars, details, _ = optunity.minimize(f,
                                         num_evals=100,
                                         x=[-5, 5],
                                         y=[-5, 5],
                                         solver_name=solver)
Exemple #4
0
def compute_mse_rbf_tuned(x_train, y_train, x_test, y_test):
    """Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""

    # define objective function for tuning
    @optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
    def tune_cv(x_train, y_train, x_test, y_test, C, gamma, epsilon):
        pipe = Pipeline([('scaler', preprocessing.StandardScaler()),
                         ('svr', SVR(C=C, gamma=gamma, epsilon=epsilon))])
        model = pipe.fit(x_train, y_train)
        predictions = model.predict(x_test)
        return optunity.metrics.mse(y_test, predictions)

    # optimize parameters
    optimal_pars, _, _ = optunity.minimize(tune_cv,
                                           150,
                                           C=[1, 100],
                                           gamma=[0, 50],
                                           epsilon=[0, 2])
    print("optimal hyperparameters: " + str(optimal_pars))

    model = Pipeline([('scaler', preprocessing.StandardScaler()),
                      ('svr', SVR(**optimal_pars))])
    tuned_model = model.fit(x_train, y_train)
    predictions = tuned_model.predict(x_test)
    return optunity.metrics.mse(y_test, predictions)
    def compute_mse_rbf_tuned(x_train, y_train, x_test, y_test):
        """Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""
        global optimal_parameters, clf
        # define objective function for tuning
        @optunity.cross_validated(x=x_train,
                                  y=y_train,
                                  num_iter=2,
                                  num_folds=2)
        def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
            # sample_weights = my_scaling_odr(y_train)
            # sample_weights = [i / max(Events[-1]) for i in Events[-1]]

            model = svm.SVR(C=C, gamma=gamma).fit(
                x_train, y_train)  #, sample_weight=sample_weights
            predictions = model.predict(x_test)
            return optunity.metrics.mse(y_test, predictions)

        # optimize parameters
        optimal_pars, _, _ = optunity.minimize(tune_cv,
                                               200,
                                               C=[1, 4000],
                                               gamma=[0, 10],
                                               pmap=optunity.pmap)
        logging.info("Optimal hyperparameters: " + str(optimal_pars))
        # sample_weights = my_scaling_odr(y_train)

        tuned_model = svm.SVR(**optimal_pars).fit(x_train, y_train)
        predictions = tuned_model.predict(x_test)
        mse = optunity.metrics.mse(y_test, predictions)
        logging.info('mse: ' + str(mse))
        if mse < mse_old:
            optimal_parameters = optimal_pars
            clf = tuned_model
        return mse
def optimize_objective(f):
    logs = {}
    solvers = optunity.available_solvers()
    for solver in solvers:
        pars, details, _ = optunity.minimize(f,
                                             num_evals=100,
                                             x=[-5, 5],
                                             y=[-5, 5],
                                             solver_name=solver)
        logs[solver] = np.array(
            [details.call_log['args']['x'], details.call_log['args']['y']])
    colors = ['r', 'g', 'b', 'y', 'k', 'y', 'r', 'g']
    markers = ['x', '+', 'o', 's', 'p', 'x', '+', 'o']

    # compute contours of the objective function
    delta = 0.025
    x = np.arange(-5.0, 5.0, delta)
    y = np.arange(-5.0, 5.0, delta)
    X, Y = np.meshgrid(x, y)
    Z = f(X, Y)
    CS = plt.contour(X, Y, Z)
    plt.clabel(CS, inline=1, fontsize=8, alpha=0.5)
    for i, solver in enumerate(solvers):
        plt.scatter(logs[solver][0, :],
                    logs[solver][1, :],
                    c=colors[i],
                    marker=markers[i],
                    alpha=0.80)
    plt.xlim([-5, 5])
    plt.ylim([-5, 5])
    plt.axis('equal')
    plt.legend(solvers)
    plt.show()
Exemple #7
0
def svr_tuned_predictions(x_train, y_train, x_test, y_test):
    @optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
    def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
        model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
        predictions = model.predict(x_test)
        return optunity.metrics.mse(y_test, predictions)

    optimal_pars, _, _ = optunity.minimize(tune_cv, 200, C=[0, 20],
                                           gamma=[0, 10], pmap=optunity.pmap)
    tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
    return tuned_model.predict(x_test).tolist()
Exemple #8
0
def initialrdf(volmax, volmin, reagent, wellnum):
    _, info_random, _ = optunity.minimize(f,
                                          num_evals=wellnum,
                                          x1=[volmin, volmax],
                                          solver_name='random search')
    ##Create quasi-random data spread
    reagentlist = info_random.call_log['args'][
        'x1']  #Create reagent amounts (mmol), change x variable to change range, each generated from unique sobol index
    reagentname = "Reagent%s (ul)" % reagent
    rdf = pd.DataFrame({reagentname: reagentlist}).astype(int)
    return (rdf)
Exemple #9
0
    def time_series():
        # the order of examples is dependent on time, and must be preserved
        # i.e. we can not randomly shuffle the data
        x_train, y_train, x_test, y_test = dataset_split(d, split=0.8)
        dictSize = 200

        def f(gamma, forget, eta, nu):
            gamma = np.exp(gamma)
            return regression(x_train, y_train, x_test, y_test, gamma, forget, eta, nu, dictSize, RP=rij)

        return optunity.minimize(f, num_evals=n_evals, gamma=[-10,0], \
                                    forget=[0.98,1], eta=[0,0.5], nu=[0, 1], solver_name=solver )
Exemple #10
0
    def normal():
        # we use a different optimizer. one which we can do cross-validation.
        X, Y, _, _ = dataset_split(d, split=1)
        dictSize = 200

        @optunity.cross_validated(x=X, y=Y, num_folds=n_folds)
        def f(x_train, y_train, x_test, y_test, gamma, forget, eta, nu):
            gamma = np.exp(gamma)
            return regression(x_train, y_train, x_test, y_test, gamma, forget, eta, nu, dictSize, RP=rij)

        return optunity.minimize(f, num_evals=n_evals, gamma=[-10,0], \
                                    forget=[0.98,1], eta=[0,0.5], nu=[0, 1], solver_name=solver )
Exemple #11
0
def svr_tuned_predictions(x_train, y_train, x_test, y_test):
    @optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
    def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
        model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
        predictions = model.predict(x_test)
        return optunity.metrics.mse(y_test, predictions)

    optimal_pars, _, _ = optunity.minimize(tune_cv,
                                           200,
                                           C=[0, 20],
                                           gamma=[0, 10],
                                           pmap=optunity.pmap)
    tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
    return tuned_model.predict(x_test).tolist()
def model_selection():
    pmap8 = optunity.parallel.create_pmap(8)
    hps, _, _ = optunity.minimize(svr_mse, num_evals=400,
                                solver_name='particle swarm', 
                                logC=[5, 7], 
                                logGamma=[-6.5, -5],
                                logEpsilon=[.7, 2],
                                # pmap=optunity.pmap)
                                pmap=pmap8)
    print(hps)
    optimal_model = sklearn.svm.SVR(C=10 ** hps['logC'],
                                gamma=10 ** hps['logGamma'], 
                                epsilon=10 ** hps['logEpsilon']
                                ).fit(features, target)
    joblib.dump(optimal_model, finalModel)
Exemple #13
0
def lr_tuned(x_train, y_train, x_test, y_test):
    @optunity.cross_validated(x=x_train, y=y_train, num_folds=3)
    def inner_cv(x_train, y_train, x_test, y_test, regularization, step):
        predict, _, _ = train_lr(x_train, y_train,
                                regularization=regularization, step=step)
        yhat = predict(x_test)
        return optunity.metrics.logloss(y_test, yhat)

    pars, _, _ = optunity.minimize(inner_cv, num_evals=50,
                                regularization=[0.001, 0.05],
                                step=[0.01, 0.2])
    predict, w, b = train_lr(x_train, y_train, **pars)
    yhat = predict(x_test)
    loss = optunity.metrics.logloss(y_test, yhat)
    brier = optunity.metrics.brier(y_test, yhat)
    return loss, brier
Exemple #14
0
    def compute_mse_svr_rbf(x_train, y_train, x_test, y_test):
        """Find the optimal hyperparameters of svm;
           Train a model using the optmal parametes
           compute MSE
        """

        # -------------------------------------------------------------------------------------------------------------
        # Find optimal parameters
        # ------------------------
        @optunity.cross_validated(x=x_train, y=y_train, num_iter=num_iter_inner_cv,
                                  num_folds=num_folds_inner_cv)
        def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
            model = SVR(C=C, gamma=gamma).fit(x_train, y_train)
            predictions = model.predict(x_test)
            return optunity.metrics.mse(y_test, predictions)

        # Optimise parameters
        optimal_pars, _, _ = optunity.minimize(tune_cv, num_evals=num_evals_inner_cv, C=C_svr_rbf, gamma=gamma_svr_rbf)
        print("THe optimal hyperparameters of SVR (kernel = rbf): " + str(optimal_pars))
        # -----------------------
        # Find optimal parameters
        # -------------------------------------------------------------------------------------------------------------

        # Train a model using the optimal parameters and the x_train and y_train
        tuned_model = SVR(**optimal_pars).fit(x_train, y_train)

        # Predict the testing data and training data
        predictions_train = tuned_model.predict(x_train)
        predictions_train = predictions_train.reshape(x_train.shape[0], order='C') # Make it one-D
        predictions_test = tuned_model.predict(x_test)
        predictions_test = predictions_test.reshape(x_test.shape[0], order='C')

        # Errors
        errors_train = errors_prediction(y_train, predictions_train)
        errors_test = errors_prediction(y_test, predictions_test)
        print('R^2_train: ', errors_train['r2_score'])
        print('R^2_test:', errors_test['r2_score'])

        # Save the parameters and errors
        predictions_labels_each_fold.append({'predictions_train': predictions_train,
                                             'labels_train': y_train,
                                             'predictions_test': predictions_test,
                                             'labels_test': y_test})
        params_each_fold.append(optimal_pars)
        errors_each_fold.append({'errors_train': errors_train, 'errors_test': errors_test})
        tuned_models_each_fold.append(tuned_model)
        return errors_test['mse']
Exemple #15
0
def compute_mse_tuned(x_train, y_train, x_test, y_test):

    # define objective function
    @optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
    def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
        model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
        predictions = model.predict(x_test)
        return optunity.metrics.mse(y_test, predictions)

    # optimize parameters
    optimal_pars, _, _ = optunity.minimize(tune_cv, 200, C=[0, 10], gamma=[0, 10], pmap=optunity.pmap)
    # if you are running this in IPython, optunity.pmap will not work
    # more info at: https://github.com/claesenm/optunity/issues/8
    # comment out the above line and replace by the one below:
    # optimal_pars, _, _ = optunity.minimize(inner_cv, 150, C=[0, 10], gamma=[0, 0.1])

    tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
    predictions = tuned_model.predict(x_test)
    return optunity.metrics.mse(y_test, predictions)
Exemple #16
0
				def compute_mse_rbf_tuned(x_train, y_train, x_test, y_test):
					"""Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""

					# define objective function for tuning
					@optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
					def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
						print("tune_cv model C="+str(C)+", gamma="+str(gamma))
						model = SVR(C=C, gamma=gamma).fit(x_train, y_train)
						print("tune_cv model fit")
						predictions = model.predict(x_test)
						return optunity.metrics.mse(y_test, predictions)

					# optimize parameters
					optimal_pars, _, _ = optunity.minimize(tune_cv, 150, C=[1, 100], gamma=[0, 50])
					print("optimal hyperparameters: " + str(optimal_pars))

					tuned_model = SVR(**optimal_pars).fit(x_train, y_train)
					predictions = tuned_model.predict(x_test)
					return optunity.metrics.mse(y_test, predictions)
Exemple #17
0
def lr_tuned(x_train, y_train, x_test, y_test):
    @optunity.cross_validated(x=x_train, y=y_train, num_folds=3)
    def inner_cv(x_train, y_train, x_test, y_test, regularization, step):
        predict, _, _ = train_lr(x_train,
                                 y_train,
                                 regularization=regularization,
                                 step=step)
        yhat = predict(x_test)
        return optunity.metrics.logloss(y_test, yhat)

    pars, _, _ = optunity.minimize(inner_cv,
                                   num_evals=50,
                                   regularization=[0.001, 0.05],
                                   step=[0.01, 0.2])
    predict, w, b = train_lr(x_train, y_train, **pars)
    yhat = predict(x_test)
    loss = optunity.metrics.logloss(y_test, yhat)
    brier = optunity.metrics.brier(y_test, yhat)
    return loss, brier
def lr_tuned(x_train, y_train, x_test, y_test):
    @optunity.cross_validated(x=x_train, y=y_train, num_folds=3)
    def inner_cv(x_train, y_train, x_test, y_test, regularization, step):
        predict, _, _ = train_lr(x_train, y_train,
                                 regularization=regularization, step=step)
        yhat = predict(x_test)
        return optunity.metrics.logloss(y_test, yhat)

    pars, _, _ = optunity.minimize(inner_cv, num_evals=50,
                                   regularization=[0.001, 0.05],
                                   step=[0.01, 0.2])
    predict, w, b = train_lr(x_train, y_train, **pars)
    yhat = predict(x_test)
    loss = optunity.metrics.logloss(y_test, yhat)
    brier = optunity.metrics.brier(y_test, yhat)
    print('+ model: ' + str(b.get_value())[:5] + ' + ' + str(w.get_value()[0])[:5] + ' * x1 + ' + str(w.get_value()[1])[:5] + ' * x2')
    print('++ log loss in test fold: ' + str(loss))
    print('++ Brier loss in test fold: ' + str(brier))
    print('')
    return loss, brier
Exemple #19
0
    def normal():
        # we use a different optimizer. one which we can do cross-validation.
        X, Y, _, _ = dataset_split(d, split=1)
        dictSize = 200

        @optunity.cross_validated(x=X, y=Y, num_folds=n_folds)
        def f(x_train, y_train, x_test, y_test, gamma, forget, eta, nu):
            gamma = np.exp(gamma)
            return regression(x_train,
                              y_train,
                              x_test,
                              y_test,
                              gamma,
                              forget,
                              eta,
                              nu,
                              dictSize,
                              RP=rij)

        return optunity.minimize(f, num_evals=n_evals, gamma=[-10,0], \
                                    forget=[0.98,1], eta=[0,0.5], nu=[0, 1], solver_name=solver )
Exemple #20
0
    def time_series():
        # the order of examples is dependent on time, and must be preserved
        # i.e. we can not randomly shuffle the data
        x_train, y_train, x_test, y_test = dataset_split(d, split=0.8)
        dictSize = 200

        def f(gamma, forget, eta, nu):
            gamma = np.exp(gamma)
            return regression(x_train,
                              y_train,
                              x_test,
                              y_test,
                              gamma,
                              forget,
                              eta,
                              nu,
                              dictSize,
                              RP=rij)

        return optunity.minimize(f, num_evals=n_evals, gamma=[-10,0], \
                                    forget=[0.98,1], eta=[0,0.5], nu=[0, 1], solver_name=solver )
def compute_mse_rbf_tuned(x_train, y_train, x_test, y_test):
    """Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""

    # define objective function for tuning
    @optunity.cross_validated(x=x_train, y=y_train,
                              num_folds=3)  #  num_iter=2,
    def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
        model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
        predictions = model.predict(x_test)
        return optunity.metrics.mse(y_test, predictions)

    # optimize parameters
    optimal_pars, _, _ = optunity.minimize(tune_cv,
                                           150,
                                           C=[55000, 65000],
                                           gamma=[0, 1])
    print("optimal hyperparameters: " + str(optimal_pars))

    tuned_model = sklearn.svm.SVR(epsilon=0.01, **optimal_pars).fit(
        x_train, y_train)  # epsilon = 0.001,
    predictions = tuned_model.predict(x_test)
    return optunity.metrics.mse(y_test, predictions)
Exemple #22
0
def compute_mse_tuned(x_train, y_train, x_test, y_test):

    # define objective function
    @optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
    def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
        model = sklearn.svm.SVR(C=C, gamma=gamma).fit(x_train, y_train)
        predictions = model.predict(x_test)
        return optunity.metrics.mse(y_test, predictions)

    # optimize parameters
    optimal_pars, _, _ = optunity.minimize(tune_cv,
                                           200,
                                           C=[0, 10],
                                           gamma=[0, 10],
                                           pmap=optunity.pmap)
    # if you are running this in IPython, optunity.pmap will not work
    # more info at: https://github.com/claesenm/optunity/issues/8
    # comment out the above line and replace by the one below:
    # optimal_pars, _, _ = optunity.minimize(inner_cv, 150, C=[0, 10], gamma=[0, 0.1])

    tuned_model = sklearn.svm.SVR(**optimal_pars).fit(x_train, y_train)
    predictions = tuned_model.predict(x_test)
    return optunity.metrics.mse(y_test, predictions)
def compute_mse_poly_tuned(x_train, y_train, x_test, y_test):
    """Computes MSE of an SVR with RBF kernel and optimized hyperparameters."""
    #
    # define objective function for tuning
    @optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
    def tune_cv(x_train, y_train, x_test, y_test, C, degree, coef0):
        model = sklearn.svm.SVR(C=C, degree=degree, coef0=coef0,
                                kernel='poly').fit(x_train, y_train)
        predictions = model.predict(x_test)
        return optunity.metrics.mse(y_test, predictions)

    #
    # optimize parameters
    optimal_pars, _, _ = optunity.minimize(tune_cv,
                                           150,
                                           C=[1000, 20000],
                                           degree=[2, 5],
                                           coef0=[0, 1])
    print("optimal hyperparameters: " + str(optimal_pars))
    #
    tuned_model = sklearn.svm.SVR(kernel='poly',
                                  **optimal_pars).fit(x_train, y_train)
    predictions = tuned_model.predict(x_test)
    return optunity.metrics.mse(y_test, predictions)
Exemple #24
0
# This file shows an example of how to retain all information during cross-validation per fold.
# For this we use the optunity.cross_validation.mean_and_list aggregator.
# For more context, cfr. https://github.com/claesenm/optunity/issues/40

import optunity
import optunity.cross_validation

x = list(range(5))


@optunity.cross_validated(x,
                          num_folds=5,
                          aggregator=optunity.cross_validation.mean_and_list)
def f(x_train, x_test, coeff):
    return x_test[0] * coeff


# evaluate f
foo = f(0.0)
print(foo)

opt_coeff, info, _ = optunity.minimize(f, coeff=[0, 1], num_evals=10)
print(opt_coeff)
print("call log")
for args, val in zip(info.call_log['args']['coeff'], info.call_log['values']):
    print(str(args) + '\t\t' + str(val))
    return(y_pred)

def rf_pso(x_train, y_train, x_valid):
    # Random forest using particle swarm opt on the hyperparameters
    # pso
    x_train = x_train.as_matrix()
    y_train = y_train.as_matrix()
    @optunity.cross_validated(x=x_train, y=y_train, num_iter=2, num_folds=5)
    def performance(x_train, y_train, x_test, y_test,n_estimators, max_features):
        model = RandomForestRegressor(n_estimators=int(n_estimators),
                                       max_features=max_features)
        model.fit(x_train, y_train)
        predictions = model.predict(x_test)
        return(optunity.metrics.mse(y_test, predictions))
    # hyperparameter search
    optimal_pars, _, _ = optunity.minimize(performance, num_evals = 100, n_estimators=[10,800], max_features=[0.2,10])
    # train
    forest_reg = RandomForestRegressor(random_state=SEED, **optimal_pars)
    reg.fit(x_train, y_train)
    # validate
    y_pred = reg.predict(x_valid)
    # return predictive probabilities
    return(y_pred)

def rf_gridsearch(x_train, y_train, x_valid):
    # Random forest using grid search
    # train
    forest_reg = RandomForestRegressor(random_state=SEED)
    param_grid = {'n_estimators': [3, 250, 500], 'max_features': [0.2, 0.4, 0.6, 0.8, 1]}
    forest_grid = GridSearchCV(forest_reg, param_grid, cv=5, scoring='mean_absolute_error')
    forest_grid.fit(x_train, y_train)
Exemple #26
0
            1]
 while (np.sum(np.sum(Y[train, :], 0) < 5) > 0):
     perm = np.random.permutation(X.shape[0])
     train = perm[0:train_samp + 1]
     test = perm[train_samp + 1:train_samp + test_samp + 1]
     val = perm[train_samp + test_samp + 1:train_samp + test_samp +
                val_samp + 1]
 prt = False
 opt_params = None
 opt_params, _, _ = opt.minimize(run_LAmbDA,
                                 solver_name='sobol',
                                 gamma=[0.8, 1.2],
                                 delta=[0.05, 0.95],
                                 tau=[1.0, 2.0],
                                 prc_cut=[20, 50],
                                 bs_prc=[0.2, 0.6],
                                 do_prc=[0.5, 1],
                                 hidden_feats=[50, 150],
                                 lambda1=[0, 5],
                                 lambda2=[3, 5],
                                 lambda3=[3, 5],
                                 num_evals=10)
 prt = True
 train = perm[0:train_samp + test_samp + 1]
 test = val
 err = run_LAmbDA(opt_params['gamma'], opt_params['delta'],
                  opt_params['tau'], opt_params['prc_cut'],
                  opt_params['bs_prc'], opt_params['do_prc'],
                  opt_params['hidden_feats'], opt_params['lambda1'],
                  opt_params['lambda2'], opt_params['lambda3'])
 tf.reset_default_graph()
def run_LAmbDA(DataPath,
               LabelsPath,
               CV_RDataPath,
               OutputDir,
               GeneOrderPath="",
               NumGenes=0):
    '''
    run LAmbDA classifier
    Wrapper script to run LAmbDA on a benchmark dataset with 5-fold cross validation,
    outputs lists of true and predicted cell labels as csv files, as well as computation time.

    Parameters
    ----------
    DataPath : Data file path (.csv), cells-genes matrix with cell unique barcodes
    as row names and gene names as column names.
    LabelsPath : Cell population annotations file path (.csv).
    CV_RDataPath : Cross validation RData file path (.RData), obtained from Cross_Validation.R function.
    OutputDir : Output directory defining the path of the exported file.
    GeneOrderPath : Gene order file path (.csv) obtained from feature selection,
    defining the genes order for each cross validation fold, default is NULL.
    NumGenes : Number of genes used in case of feature selection (integer), default is 0.
    '''

    # read the Rdata file
    robjects.r['load'](CV_RDataPath)

    nfolds = np.array(robjects.r['n_folds'], dtype='int')
    tokeep = np.array(robjects.r['Cells_to_Keep'], dtype='bool')
    col = np.array(robjects.r['col_Index'], dtype='int')
    col = col - 1
    test_ind = np.array(robjects.r['Test_Idx'])
    train_ind = np.array(robjects.r['Train_Idx'])

    # read the data
    data = pd.read_csv(DataPath, index_col=0, sep=',')
    labels = pd.read_csv(LabelsPath,
                         header=0,
                         index_col=None,
                         sep=',',
                         usecols=col)

    labels = labels.iloc[tokeep]
    data = data.iloc[tokeep]

    # read the feature file
    if (NumGenes > 0):
        features = pd.read_csv(GeneOrderPath,
                               header=0,
                               index_col=None,
                               sep=',')

    # folder with results
    os.chdir(OutputDir)

    tr_time = []
    ts_time = []
    truelab = np.zeros([len(labels), 1], dtype=int)
    predlab = np.zeros([len(labels), 1], dtype=int)

    for i in range(np.squeeze(nfolds)):
        global X, Y, Gnp, Dnp, train, test, prt, cv
        test_ind_i = np.array(test_ind[i], dtype='int') - 1
        train_ind_i = np.array(train_ind[i], dtype='int') - 1

        X = np.array(data)
        if (NumGenes > 0):
            X = np.log2(X / 10 + 1)
            feat_to_use = features.iloc[0:NumGenes, i]
            X = X[:, feat_to_use]
        else:
            X = np.log2(
                np.transpose(select_feats(np.transpose(X), 0.5, 80)) / 10 + 1)

        uniq = np.unique(labels)
        Y = np.zeros([len(labels), len(uniq)], int)

        for j in range(len(uniq)):
            Y[np.where(labels == uniq[j])[0], j] = 1

        Y = np.array(Y)

        Gnp = np.zeros([len(uniq), len(uniq)], int)
        np.fill_diagonal(Gnp, 1)
        Gnp = np.array(Gnp)

        Dnp = np.ones([len(uniq), 1], int)
        Dnp = np.array(Dnp)

        train_samp = int(np.floor(0.75 * len(train_ind_i)))
        test_samp = len(train_ind_i) - train_samp
        perm = np.random.permutation(len(train_ind_i))
        train = perm[0:train_samp]
        test = perm[train_samp:test_samp + 1]

        while (np.sum(np.sum(Y[train, :], 0) < 5) > 0):
            perm = np.random.permutation(X.shape[0])
            train = perm[0:train_samp + 1]
            test = perm[train_samp + 1:train_samp + test_samp + 1]

        cv = i
        optunity_it = 0
        prt = False
        opt_params = None

        start = tm.time()
        opt_params, _, _ = opt.minimize(run_LAmbDA2,
                                        solver_name='sobol',
                                        gamma=[0.8, 1.2],
                                        delta=[0.05, 0.95],
                                        tau=[10.0, 11.0],
                                        prc_cut=[20, 50],
                                        bs_prc=[0.2, 0.6],
                                        num_trees=[10, 200],
                                        max_nodes=[100, 1000],
                                        num_evals=50)
        tr_time.append(tm.time() - start)

        print("Finished training!")

        prt = True
        train = train_ind_i
        test = test_ind_i

        start = tm.time()
        err = run_LAmbDA2(opt_params['gamma'], opt_params['delta'],
                          opt_params['tau'], opt_params['prc_cut'],
                          opt_params['bs_prc'], opt_params['num_trees'],
                          opt_params['max_nodes'])
        ts_time.append(tm.time() - start)

        tf.reset_default_graph()

        predfile = 'preds_cv' + str(cv) + '.mat'
        truefile = 'truth_cv' + str(cv) + '.mat'
        pred = sio.loadmat(predfile)
        truth = sio.loadmat(truefile)

        pred = pred['preds']
        truth = truth['labels']

        pred_ind = np.argmax(pred, axis=1)
        truth_ind = np.argmax(truth, axis=1)

        predlab[test_ind_i, 0] = pred_ind
        truelab[test_ind_i, 0] = truth_ind

    truelab = pd.DataFrame(truelab)
    predlab = pd.DataFrame(predlab)

    tr_time = pd.DataFrame(tr_time)
    ts_time = pd.DataFrame(ts_time)

    if (NumGenes == 0):
        truelab.to_csv("LAmbDA_True_Labels.csv", index=False)
        predlab.to_csv("LAmbDA_Pred_Labels.csv", index=False)
        tr_time.to_csv("LAmbDA_Training_Time.csv", index=False)
        ts_time.to_csv("LAmbDA_Testing_Time.csv", index=False)
    else:
        truelab.to_csv("LAmbDA_" + str(NumGenes) + "_True_Labels.csv",
                       index=False)
        predlab.to_csv("LAmbDA_" + str(NumGenes) + "_Pred_Labels.csv",
                       index=False)
        tr_time.to_csv("LAmbDA_" + str(NumGenes) + "_Training_Time.csv",
                       index=False)
        ts_time.to_csv("LAmbDA_" + str(NumGenes) + "_Testing_Time.csv",
                       index=False)
    def search_param(self, database, dataprocess=None, path_filename=("", ""),
                     save=False, cv="ts", of="rmse", f=None, eval=50):
        """
            Search best hyperparameters for classifier/regressor based on
            optunity algorithms.

            Arguments:
                database (numpy.ndarray): a matrix containing all patterns
                    that will be used for training/testing at some
                    cross-validation method.
                dataprocess (DataProcess): an object that will pre-process
                    database before training. Defaults to None.
                path_filename (tuple): *TODO*.
                save (bool): *TODO*.
                cv (str): Cross-validation method. Defaults to "ts".
                of (str): Objective function to be minimized at
                    optunity.minimize. Defaults to "rmse".
                f (list of str): a list of functions to be used by the
                    search. Defaults to None, this set all available
                    functions.
                eval (int): Number of steps (evaluations) to optunity algorithm.

            Each set of hyperparameters will perform a cross-validation
            method chosen by param cv.

            Available *cv* methods:
                - "ts" :func:`mltools.time_series_cross_validation()`
                    Perform a time-series cross-validation suggested by Hydman.

                - "kfold" :func:`mltools.kfold_cross_validation()`
                    Perform a k-fold cross-validation.

            Available *of* function:
                - "accuracy", "rmse", "mape", "me".


            See Also:
                http://optunity.readthedocs.org/en/latest/user/index.html
        """

        if f is None:
            search_functions = self.available_functions
        elif type(f) is list:
            search_functions = f
        else:
            raise Exception("Invalid format for argument 'f'.")

        print(self.regressor_name)
        print("##### Start search #####")

        config = configparser.ConfigParser()
        if sys.version_info < (3, 0):
            config.readfp(open(_ELMR_CONFIG))
        else:
            config.read_file(open(_ELMR_CONFIG))

        best_function_error = 99999.9
        temp_error = best_function_error
        best_param_function = ""
        best_param_c = 0
        best_param_l = 0
        for function in search_functions:

            if sys.version_info < (3, 0):
                elmr_c_range = ast.literal_eval(config.get("DEFAULT",
                                                           "elmr_c_range"))

                neurons = config.getint("DEFAULT", "elmr_neurons")

            else:
                function_config = config["DEFAULT"]
                elmr_c_range = ast.literal_eval(function_config["elmr_c_range"])
                neurons = ast.literal_eval(function_config["elmr_neurons"])

            param_ranges = [[elmr_c_range[0][0], elmr_c_range[0][1]]]

            def wrapper_opt(param_c):
                """
                    Wrapper for optunity.
                """

                if cv == "ts":
                    cv_tr_error, cv_te_error = \
                        time_series_cross_validation(self, database,
                                                     params=[function,
                                                             2 ** param_c,
                                                             neurons,
                                                             False],
                                                     number_folds=10,
                                                     dataprocess=dataprocess)

                elif cv == "kfold":
                    cv_tr_error, cv_te_error = \
                        kfold_cross_validation(self, database,
                                               params=[function,
                                                       2 ** param_c,
                                                       neurons,
                                                       False],
                                               number_folds=10,
                                               dataprocess=dataprocess)

                else:
                    raise Exception("Invalid type of cross-validation.")

                if of == "accuracy":
                    util = 1 / cv_te_error.get_accuracy()
                else:
                    util = cv_te_error.get(of)

                # print("c:", param_c, "util: ", util)
                return util

            optimal_pars, details, _ =  \
                optunity.minimize(wrapper_opt,
                                  solver_name="cma-es",
                                  num_evals=eval,
                                  param_c=param_ranges[0])

            # Save best function result
            if details[0] < temp_error:
                temp_error = details[0]

                if of == "accuracy":
                    best_function_error = 1 / temp_error
                else:
                    best_function_error = temp_error

                best_param_function = function
                best_param_c = optimal_pars["param_c"]
                best_param_l = neurons

            if of == "accuracy":
                print("Function: ", function,
                      " best cv value: ", 1/details[0])
            else:
                print("Function: ", function,
                      " best cv value: ", details[0])

        # MLTools Attribute
        self.cv_best_rmse = best_function_error

        # elmr Attribute
        self.param_function = best_param_function
        self.param_c = best_param_c
        self.param_l = best_param_l

        print("##### Search complete #####")
        self.print_parameters()

        return None
Exemple #29
0
def main():

    parser = argparse.ArgumentParsers(
        description="Run grid optimization on a single subject")
    parser.add_argument('msh',
                        type=str,
                        help="Subject Gmsh .msh realistic head model")
    parser.add_argument('weights',
                        type=str,
                        help=".npy binary containing a weight for each "
                        "tetrahedron")
    parser.add_argument('centroid',
                        type=str,
                        help="Coordinates in T1w space for a centroid "
                        "to the weight function to optimize over")
    parser.add_argument('coil', type=str, help="Path to SimNIBS coil file")
    parser.add_argument('output_file',
                        type=str,
                        help="Output file storing optimal coordinates")
    parser.add_argument('output_file',
                        type=str,
                        help="Output file storing optimal coordinates")
    parser.add_argument('--history',
                        type=str,
                        help="Output file to store history of scores"
                        " into for convergence/visualization")
    parser.add_argument('--workdir',
                        type=str,
                        help="Working directory to run simulations in")
    parser.add_argument('--ncpus',
                        type=int,
                        help="Number of threads to use for each batch "
                        "of simulations. Default = 8")
    parser.add_argument('--batchsize',
                        type=int,
                        help="Number of simulations to run simultaneously, "
                        "will default to half the number of cpus if not "
                        "specified.")
    parser.add_argument('--solver',
                        type=int,
                        help="Optunity solver to use, "
                        "defaults to particle swarm",
                        choices=optunity.available_solvers())

    args = parser.parse()
    msh = args.msh
    wf = np.load(args.weights)
    centroid = np.genfromtxt(args.centroid)
    coil = args.coil
    ncpus = args.ncpus or 8
    batch_size = args.batchsize or (ncpus // 2 - 1)
    history = args.history
    workdir = args.workdir or "/tmp/"
    output_file = args.output_file
    solver = args.solver or "particle swarm"

    # Construct objective function object
    f = FieldFunc(mesh_file=msh,
                  initial_centroid=centroid,
                  tet_weights=wf,
                  coil=coil,
                  field_dir=workdir,
                  cpus=ncpus)

    # Set up optunity optimization
    # Can we feed a list of inputs here?
    pars, details, _ = optunity.minimize(f.evaluate,
                                         num_evals=100,
                                         x=[f.bounds[0, 0], f.bounds[0, 1]],
                                         y=[f.bounds[1, 0], f.bounds[1, 1]],
                                         theta=[0, 180],
                                         solver_name=solver)
Exemple #30
0
def modelling_svr_rbf(C_svr_rbf,
                      gamma_svr_rbf,
                      wavelengths_range,
                      input_type,
                      num_folds_outer_cv,
                      num_iter_inner_cv,
                      num_folds_inner_cv,
                      num_evals_inner_cv,
                      samples,
                      wavelengths,
                      labels,
                      flag_save,
                      flag_fig):
    """ Model a svr with rbf kernel."""

    start = datetime.datetime.now()
    print('')
    print('svr (kernel = rbf)')
    print('The range of C is: ', C_svr_rbf)
    print('The range of gamma is: ', gamma_svr_rbf)
    print('')

    # For records
    date_time = datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S')
    model_name = 'svr_rbf'
    save_record_name = 'record' + '_' + wavelengths_range + '_' + input_type + '_' + model_name + '.sav'
    save_model_name = 'model' + '_' + wavelengths_range + '_' + input_type + '_' + model_name + '.sav'

    # //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    # CV
    # ///
    print('Conducting cross-validation')

    # For record
    params_each_fold = []
    errors_each_fold = []
    predictions_labels_each_fold = []
    tuned_models_each_fold = []

    # ==================================================================================================================
    # Thf function for outer_cv
    # ==========================
    def compute_mse_svr_rbf(x_train, y_train, x_test, y_test):
        """Find the optimal hyperparameters of svm;
           Train a model using the optmal parametes
           compute MSE
        """

        # -------------------------------------------------------------------------------------------------------------
        # Find optimal parameters
        # ------------------------
        @optunity.cross_validated(x=x_train, y=y_train, num_iter=num_iter_inner_cv,
                                  num_folds=num_folds_inner_cv)
        def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
            model = SVR(C=C, gamma=gamma).fit(x_train, y_train)
            predictions = model.predict(x_test)
            return optunity.metrics.mse(y_test, predictions)

        # Optimise parameters
        optimal_pars, _, _ = optunity.minimize(tune_cv, num_evals=num_evals_inner_cv, C=C_svr_rbf, gamma=gamma_svr_rbf)
        print("THe optimal hyperparameters of SVR (kernel = rbf): " + str(optimal_pars))
        # -----------------------
        # Find optimal parameters
        # -------------------------------------------------------------------------------------------------------------

        # Train a model using the optimal parameters and the x_train and y_train
        tuned_model = SVR(**optimal_pars).fit(x_train, y_train)

        # Predict the testing data and training data
        predictions_train = tuned_model.predict(x_train)
        predictions_train = predictions_train.reshape(x_train.shape[0], order='C') # Make it one-D
        predictions_test = tuned_model.predict(x_test)
        predictions_test = predictions_test.reshape(x_test.shape[0], order='C')

        # Errors
        errors_train = errors_prediction(y_train, predictions_train)
        errors_test = errors_prediction(y_test, predictions_test)
        print('R^2_train: ', errors_train['r2_score'])
        print('R^2_test:', errors_test['r2_score'])

        # Save the parameters and errors
        predictions_labels_each_fold.append({'predictions_train': predictions_train,
                                             'labels_train': y_train,
                                             'predictions_test': predictions_test,
                                             'labels_test': y_test})
        params_each_fold.append(optimal_pars)
        errors_each_fold.append({'errors_train': errors_train, 'errors_test': errors_test})
        tuned_models_each_fold.append(tuned_model)
        return errors_test['mse']
    # =========================
    # The function for outer cv
    # ==================================================================================================================

    # The fellow is the same as:
    # @optunity.cross_validated(x=samples, y=labels, num_folds=num_folds_outer_cv)
    # def compute_mse_svr_rbf:
    #     ...
    #
    # compute_mse_svr_rbf()
    outer_cv = optunity.cross_validated(x=samples, y=labels, num_folds=num_folds_outer_cv)  # function decoter
    compute_mse_svr_rbf = outer_cv(compute_mse_svr_rbf) # Decorate computer_mse_svr_rbf
    compute_mse_svr_rbf()

    print('The cross-validation has been done!', datetime.datetime.now().strftime('%y-%m-%d-%H-%M-%S'))
    stop = datetime.datetime.now()
    print('Total time used ', stop - start)
    # ///
    # CV
    # //////////////////////////////////////////////////////////////////////////////////////////////////////////////////

    # Record the results
    ave_errors = errors_average(errors_each_fold)
    record_svr_rbf = {'model_name': save_model_name,
                      'date_time': date_time,
                      'C_range': C_svr_rbf,
                      'gamma_range': gamma_svr_rbf,
                      'num_folds_outer_cv': num_folds_outer_cv,
                      'num_iter_inner_cv': num_iter_inner_cv,
                      'num_folds_inner_cv': num_folds_inner_cv,
                      'num_evals_inner_cv': num_evals_inner_cv,
                      'tuned_models_each_fold': tuned_models_each_fold,
                      'predictions_labels_each_fold': predictions_labels_each_fold,
                      'optimal_parameters_each_fold': params_each_fold,
                      'errors_each_fold': errors_each_fold,
                      'average_errors': ave_errors,
                      'wavelengths': wavelengths
                      }

    # Print average of cv
    print_ave_errors_cv(ave_errors)

    if flag_fig:
        # Plot a record in one (random selected) of the cv
        plot_regression_result(predictions_labels_each_fold[0]['labels_train'],
                               predictions_labels_each_fold[0]['predictions_train'])
        plot_regression_result(predictions_labels_each_fold[0]['labels_test'],
                               predictions_labels_each_fold[0]['predictions_test'])


    # //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    # Train a model using all of the data
    # ////////////////////////////////////
    # ==================================================================================================================
    # Find the optimal parameters
    # ============================
    print('Training a SVR (kernel = rbf) instance.')
    @optunity.cross_validated(x=samples, y=labels, num_iter=num_iter_inner_cv,
                              num_folds=num_folds_inner_cv)
    def tune_cv(x_train, y_train, x_test, y_test, C, gamma):
        model = SVR(C=C, gamma=gamma).fit(x_train, y_train)
        predictions = model.predict(x_test)
        return optunity.metrics.mse(y_test, predictions)


    # Optimise parameters
    optimal_pars, _, _ = optunity.minimize(tune_cv, num_evals=num_evals_inner_cv, C=C_svr_rbf, gamma=gamma_svr_rbf)
    # ============================
    # Find the optimal parameters
    # ==================================================================================================================

    # Train a model using all of the data
    tuned_model_finial = SVR(**optimal_pars).fit(samples, labels)
    # ///////////////////////////////////
    # Train a model using all of the data
    # //////////////////////////////////////////////////////////////////////////////////////////////////////////////////

    # Save the model
    if flag_save:
        joblib.dump(record_svr_rbf, save_record_name)
        joblib.dump(tuned_model_finial, save_model_name)
        print('The tuned_model_finial and the record has been saved!')

    return record_svr_rbf, tuned_model_finial
    # SVM regression with rbf kernel
################################
# SVM regression with rbf kernel
########################################################################################################################
Exemple #31
0
    return optunity.metrics.roc_auc(y_test, predictions, positive=True)
    

performance(algorithm='k-nn', n_neighbors=3)


optimal_configuration, info, _ = optunity.maximize_structured(performance,
                                                              search_space=search,
                                                              num_evals=300)
                                            
print(optimal_configuration)
print(info.optimum)                                         
solution = dict([(k, v) for k, v in optimal_configuration.items() if v is not None])
print('Solution\n========')
print("\n".join(map(lambda x: "%s \t %s" % (x[0], str(x[1])), solution.items())))

#basic optim
def create_objective_function():
    xoff = random.random()
    yoff = random.random()
    def f(x, y):
        return (x - xoff)**2 + (y - yoff)**2
    return f
solvers = optunity.available_solvers()
print('Available solvers: ' + ', '.join(solvers))
f = create_objective_function()
logs = {}
for solver in solvers:
    pars, details, _ = optunity.minimize(f, num_evals=100, x=[-5, 5], y=[-5, 5],solver_name=solver)
    logs[solver] = np.array([details.call_log['args']['x'],
                             details.call_log['args']['y']])
Exemple #32
0
# This file shows an example of how to retain all information during cross-validation per fold.
# For this we use the optunity.cross_validation.mean_and_list aggregator.
# For more context, cfr. https://github.com/claesenm/optunity/issues/40

import optunity
import optunity.cross_validation

x = list(range(5))

@optunity.cross_validated(x, num_folds=5,
                          aggregator=optunity.cross_validation.mean_and_list)
def f(x_train, x_test, coeff):
    return x_test[0] * coeff

# evaluate f
foo = f(0.0)
print(foo)

opt_coeff, info, _ = optunity.minimize(f, coeff=[0, 1], num_evals=10)
print(opt_coeff)
print("call log")
for args, val in zip(info.call_log['args']['coeff'], info.call_log['values']):
    print(str(args) + '\t\t' + str(val))
Exemple #33
0
    def fit(self,
            obj_fun=None,
            param_range=None,
            num_evals=1000,
            solver_name="sobol",
            maximize=True,
            num_jobs=1,
            engine="optunity",
            random_state=1,
            init_points=20,
            n_iter=50,
            acq='ucb',
            kappa=2.576,
            xi=0.0):
        if engine == "optunity":
            if obj_fun:
                self.obj_fun = obj_fun
            if param_range:
                self.param_range = param_range
            if num_jobs == 1:
                if maximize:
                    self.optimal_params, self.info, _ = op.maximize(
                        self.obj_fun,
                        num_evals=num_evals,
                        solver_name=solver_name,
                        **self.param_range)  # default: 'particle swarm'
                else:
                    self.optimal_params, self.info, _ = op.minimize(
                        self.obj_fun,
                        num_evals=num_evals,
                        solver_name=solver_name,
                        **self.param_range)  # de
#                 params
#                 print(self.optimal_params)
#                 optimum and states
                print(self.info.optimum, self.info.stats)
            elif num_jobs > 1:
                if maximize:
                    self.optimal_params, self.info, _ = op.maximize(
                        self.obj_fun,
                        num_evals=num_evals,
                        solver_name=solver_name,
                        pmap=op.parallel.create_pmap(num_jobs),
                        **self.param_range)  # default: 'particle swarm'
                else:
                    self.optimal_params, self.info, _ = op.minimize(
                        self.obj_fun,
                        num_evals=num_evals,
                        solver_name=solver_name,
                        pmap=op.parallel.create_pmap(num_jobs),
                        **self.param_range)  # de
                # params


#                 print(self.optimal_params)
# optimum and states
                print(self.info.optimum, self.info.stats)
        elif engine == "bayes_opt":
            if obj_fun:
                self.obj_fun = obj_fun
            if param_range:
                self.param_range = param_range
            param_range_t = {}
            for key in self.param_range:
                param_range_t[key] = tuple(self.param_range[key])
            optimizer = BayesianOptimization(
                f=self.obj_fun,
                pbounds=param_range_t,
                random_state=random_state,
            )
            optimizer.maximize(
                init_points=init_points,
                n_iter=n_iter,
                acq=acq,
                kappa=kappa,
                xi=xi,
            )
            self.optimal_params = optimizer.max['params']
 val = perm[train_samp + test_samp + 1:train_samp + test_samp + val_samp +
            1]
 while (np.sum(np.sum(Y[train, :], 0) < 5) > 0):
     perm = np.random.permutation(X.shape[0])
     train = perm[0:train_samp + 1]
     test = perm[train_samp + 1:train_samp + test_samp + 1]
     val = perm[train_samp + test_samp + 1:train_samp + test_samp +
                val_samp + 1]
 optunity_it = 0
 prt = False
 opt_params = None
 opt_params, _, _ = opt.minimize(run_LAmbDA,
                                 solver_name='sobol',
                                 gamma=[0.8, 1.2],
                                 delta=[0.05, 0.95],
                                 tau=[10.0, 11.0],
                                 prc_cut=[20, 50],
                                 bs_prc=[0.2, 0.6],
                                 num_trees=[10, 200],
                                 max_nodes=[100, 1000],
                                 num_evals=50)
 print(opt_params)
 prt = True
 train = perm[0:train_samp + test_samp + 1]
 test = val
 err = run_LAmbDA(opt_params['gamma'], opt_params['delta'],
                  opt_params['tau'], opt_params['prc_cut'],
                  opt_params['bs_prc'], opt_params['num_trees'],
                  opt_params['max_nodes'])
 tf.reset_default_graph()
 print('Cross validation step: ' + str(cv))
 print(opt_params)
Exemple #35
0
    def search_param(self, database, dataprocess=None, path_filename=("", ""),
                     save=False, cv="ts", of="rmse", kf=None, eval=50):
        """
            Search best hyperparameters for classifier/regressor based on
            optunity algorithms.

            Arguments:
                database (numpy.ndarray): a matrix containing all patterns
                    that will be used for training/testing at some
                    cross-validation method.
                dataprocess (DataProcess): an object that will pre-process
                    database before training. Defaults to None.
                path_filename (tuple): *TODO*.
                save (bool): *TODO*.
                cv (str): Cross-validation method. Defaults to "ts".
                of (str): Objective function to be minimized at
                    optunity.minimize. Defaults to "rmse".
                kf (list of str): a list of kernel functions to be used by
                    the search. Defaults to None, this set all available
                    functions.
                eval (int): Number of steps (evaluations) to optunity algorithm.


            Each set of hyperparameters will perform a cross-validation
            method chosen by param cv.


            Available *cv* methods:
                - "ts" :func:`mltools.time_series_cross_validation()`
                    Perform a time-series cross-validation suggested by Hydman.

                - "kfold" :func:`mltools.kfold_cross_validation()`
                    Perform a k-fold cross-validation.

            Available *of* function:
                - "accuracy", "rmse", "mape", "me".


            See Also:
                http://optunity.readthedocs.org/en/latest/user/index.html
        """

        if kf is None:
            search_kernel_functions = self.available_kernel_functions
        elif type(kf) is list:
            search_kernel_functions = kf
        else:
            raise Exception("Invalid format for argument 'kf'.")

        print(self.regressor_name)
        print("##### Start search #####")

        config = configparser.ConfigParser()

        if sys.version_info < (3, 0):
            config.readfp(open(_ELMK_CONFIG))
        else:
            config.read_file(open(_ELMK_CONFIG))

        best_function_error = 99999.9
        temp_error = best_function_error
        best_param_c = 0
        best_param_kernel_function = ""
        best_param_kernel_param = []
        for kernel_function in search_kernel_functions:

            if sys.version_info < (3, 0):
                elmk_c_range = ast.literal_eval(config.get("DEFAULT",
                                                           "elmk_c_range"))

                n_parameters = config.getint(kernel_function, "kernel_n_param")
                kernel_p_range = \
                    ast.literal_eval(config.get(kernel_function,
                                                "kernel_params_range"))

            else:
                kernel_config = config[kernel_function]

                elmk_c_range = ast.literal_eval(kernel_config["elmk_c_range"])

                n_parameters = int(kernel_config["kernel_n_param"])
                kernel_p_range = \
                    ast.literal_eval(kernel_config["kernel_params_range"])

            param_ranges = [[elmk_c_range[0][0], elmk_c_range[0][1]]]
            for param in range(n_parameters):
                    param_ranges.append([kernel_p_range[param][0],
                                         kernel_p_range[param][1]])

            def wrapper_0param(param_c):
                """
                    Wrapper for objective function.
                """

                if cv == "ts":
                    cv_tr_error, cv_te_error = \
                        time_series_cross_validation(self, database,
                                                     [kernel_function,
                                                      param_c,
                                                      list([])],
                                                     number_folds=10,
                                                     dataprocess=dataprocess)

                elif cv == "kfold":
                    cv_tr_error, cv_te_error = \
                        kfold_cross_validation(self, database,
                                               [kernel_function,
                                                param_c,
                                                list([])],
                                               number_folds=10,
                                               dataprocess=dataprocess)

                else:
                    raise Exception("Invalid type of cross-validation.")

                if of == "accuracy":
                    util = 1 / cv_te_error.get_accuracy()
                else:
                    util = cv_te_error.get(of)

                # print("c:", param_c, "util: ", util)
                return util

            def wrapper_1param(param_c, param_kernel):
                """
                    Wrapper for optunity.
                """

                if cv == "ts":
                    cv_tr_error, cv_te_error = \
                        time_series_cross_validation(self, database,
                                                     [kernel_function,
                                                      param_c,
                                                      list([param_kernel])],
                                                     number_folds=10,
                                                     dataprocess=dataprocess)

                elif cv == "kfold":
                    cv_tr_error, cv_te_error = \
                        kfold_cross_validation(self, database,
                                               [kernel_function,
                                                param_c,
                                                list([param_kernel])],
                                               number_folds=10,
                                               dataprocess=dataprocess)

                else:
                    raise Exception("Invalid type of cross-validation.")

                if of == "accuracy":
                    util = 1 / cv_te_error.get_accuracy()
                else:
                    util = cv_te_error.get(of)

                # print("c:", param_c, " gamma:", param_kernel, "util: ", util)
                return util

            def wrapper_2param(param_c, param_kernel1, param_kernel2):
                """
                    Wrapper for optunity.
                """

                if cv == "ts":
                    cv_tr_error, cv_te_error = \
                        time_series_cross_validation(self, database,
                                                     [kernel_function,
                                                      param_c,
                                                      list([param_kernel1,
                                                            param_kernel2])],
                                                     number_folds=10,
                                                     dataprocess=dataprocess)

                elif cv == "kfold":
                    cv_tr_error, cv_te_error = \
                        kfold_cross_validation(self, database,
                                               [kernel_function,
                                                param_c,
                                                list([param_kernel1,
                                                      param_kernel2])],
                                               number_folds=10,
                                               dataprocess=dataprocess)

                else:
                    raise Exception("Invalid type of cross-validation.")

                if of == "accuracy":
                    util = 1 / cv_te_error.get_accuracy()
                else:
                    util = cv_te_error.get(of)

                # print("c:", param_c, " param1:", param_kernel1,
                #       " param2:", param_kernel2, "util: ", util)
                return util

            if kernel_function == "linear":
                optimal_parameters, details, _ = \
                    optunity.minimize(wrapper_0param,
                                      solver_name="cma-es",
                                      num_evals=eval,
                                      param_c=param_ranges[0])

            elif kernel_function == "rbf":
                optimal_parameters, details, _ = \
                    optunity.minimize(wrapper_1param,
                                      solver_name="cma-es",
                                      num_evals=eval,
                                      param_c=param_ranges[0],
                                      param_kernel=param_ranges[1])

            elif kernel_function == "poly":
                optimal_parameters, details, _ = \
                    optunity.minimize(wrapper_2param,
                                      solver_name="cma-es",
                                      num_evals=eval,
                                      param_c=param_ranges[0],
                                      param_kernel1=param_ranges[1],
                                      param_kernel2=param_ranges[2])
            else:
                raise Exception("Invalid kernel function.")

            # Save best kernel result
            if details[0] < temp_error:
                temp_error = details[0]

                if of == "accuracy":
                    best_function_error = 1 / temp_error
                else:
                    best_function_error = temp_error

                best_param_kernel_function = kernel_function
                best_param_c = optimal_parameters["param_c"]

                if best_param_kernel_function == "linear":
                    best_param_kernel_param = []
                elif best_param_kernel_function == "rbf":
                    best_param_kernel_param = [optimal_parameters["param_kernel"]]
                elif best_param_kernel_function == "poly":
                    best_param_kernel_param = \
                        [optimal_parameters["param_kernel1"],
                         optimal_parameters["param_kernel2"]]
                else:
                    raise Exception("Invalid kernel function.")

                # print("best: ", best_param_kernel_function,
                #       best_function_error, best_param_c, best_param_kernel_param)

            if of == "accuracy":
                print("Kernel function: ", kernel_function,
                      " best cv value: ", 1/details[0])
            else:
                print("Kernel function: ", kernel_function,
                      " best cv value: ", details[0])


        # MLTools attribute
        self.cv_best_rmse = best_function_error

        # ELM attribute
        self.param_c = best_param_c
        self.param_kernel_function = best_param_kernel_function
        self.param_kernel_params = best_param_kernel_param

        print("##### Search complete #####")
        self.print_parameters()

        return None
Exemple #36
0
    def search_param(self,
                     database,
                     dataprocess=None,
                     path_filename=("", ""),
                     save=False,
                     cv="ts",
                     of="rmse",
                     f=None,
                     eval=50):
        """
            Search best hyperparameters for classifier/regressor based on
            optunity algorithms.

            Arguments:
                database (numpy.ndarray): a matrix containing all patterns
                    that will be used for training/testing at some
                    cross-validation method.
                dataprocess (DataProcess): an object that will pre-process
                    database before training. Defaults to None.
                path_filename (tuple): *TODO*.
                save (bool): *TODO*.
                cv (str): Cross-validation method. Defaults to "ts".
                of (str): Objective function to be minimized at
                    optunity.minimize. Defaults to "rmse".
                f (list of str): a list of functions to be used by the
                    search. Defaults to None, this set all available
                    functions.
                eval (int): Number of steps (evaluations) to optunity algorithm.

            Each set of hyperparameters will perform a cross-validation
            method chosen by param cv.

            Available *cv* methods:
                - "ts" :func:`mltools.time_series_cross_validation()`
                    Perform a time-series cross-validation suggested by Hydman.

                - "kfold" :func:`mltools.kfold_cross_validation()`
                    Perform a k-fold cross-validation.

            Available *of* function:
                - "accuracy", "rmse", "mape", "me".


            See Also:
                http://optunity.readthedocs.org/en/latest/user/index.html
        """

        if f is None:
            search_functions = self.available_functions
        elif type(f) is list:
            search_functions = f
        else:
            raise Exception("Invalid format for argument 'f'.")

        print(self.regressor_name)
        print("##### Start search #####")

        config = configparser.ConfigParser()
        if sys.version_info < (3, 0):
            config.readfp(open(_ELMR_CONFIG))
        else:
            config.read_file(open(_ELMR_CONFIG))

        best_function_error = 99999.9
        temp_error = best_function_error
        best_param_function = ""
        best_param_c = 0
        best_param_l = 0
        for function in search_functions:

            if sys.version_info < (3, 0):
                elmr_c_range = ast.literal_eval(
                    config.get("DEFAULT", "elmr_c_range"))

                neurons = config.getint("DEFAULT", "elmr_neurons")

            else:
                function_config = config["DEFAULT"]
                elmr_c_range = ast.literal_eval(
                    function_config["elmr_c_range"])
                neurons = ast.literal_eval(function_config["elmr_neurons"])

            param_ranges = [[elmr_c_range[0][0], elmr_c_range[0][1]]]

            def wrapper_opt(param_c):
                """
                    Wrapper for optunity.
                """

                if cv == "ts":
                    cv_tr_error, cv_te_error = \
                        time_series_cross_validation(self, database,
                                                     params=[function,
                                                             2 ** param_c,
                                                             neurons,
                                                             False],
                                                     number_folds=10,
                                                     dataprocess=dataprocess)

                elif cv == "kfold":
                    cv_tr_error, cv_te_error = \
                        kfold_cross_validation(self, database,
                                               params=[function,
                                                       2 ** param_c,
                                                       neurons,
                                                       False],
                                               number_folds=10,
                                               dataprocess=dataprocess)

                else:
                    raise Exception("Invalid type of cross-validation.")

                if of == "accuracy":
                    util = 1 / cv_te_error.get_accuracy()
                else:
                    util = cv_te_error.get(of)

                # print("c:", param_c, "util: ", util)
                return util

            optimal_pars, details, _ =  \
                optunity.minimize(wrapper_opt,
                                  solver_name="cma-es",
                                  num_evals=eval,
                                  param_c=param_ranges[0])

            # Save best function result
            if details[0] < temp_error:
                temp_error = details[0]

                if of == "accuracy":
                    best_function_error = 1 / temp_error
                else:
                    best_function_error = temp_error

                best_param_function = function
                best_param_c = optimal_pars["param_c"]
                best_param_l = neurons

            if of == "accuracy":
                print("Function: ", function, " best cv value: ",
                      1 / details[0])
            else:
                print("Function: ", function, " best cv value: ", details[0])

        # MLTools Attribute
        self.cv_best_rmse = best_function_error

        # elmr Attribute
        self.param_function = best_param_function
        self.param_c = best_param_c
        self.param_l = best_param_l

        print("##### Search complete #####")
        self.print_parameters()

        return None
Exemple #37
0
print(info.optimum)
solution = dict([(k, v) for k, v in optimal_configuration.items()
                 if v is not None])
print('Solution\n========')
print("\n".join(map(lambda x: "%s \t %s" % (x[0], str(x[1])),
                    solution.items())))


#basic optim
def create_objective_function():
    xoff = random.random()
    yoff = random.random()

    def f(x, y):
        return (x - xoff)**2 + (y - yoff)**2

    return f


solvers = optunity.available_solvers()
print('Available solvers: ' + ', '.join(solvers))
f = create_objective_function()
logs = {}
for solver in solvers:
    pars, details, _ = optunity.minimize(f,
                                         num_evals=100,
                                         x=[-5, 5],
                                         y=[-5, 5],
                                         solver_name=solver)
    logs[solver] = np.array(
        [details.call_log['args']['x'], details.call_log['args']['y']])
Exemple #38
0
                crps_train_min = crps_train

        print('Saving weights...')
        # save weights so they can be loaded later
        model_systole.save_weights('weights_systole.hdf5', overwrite=True)
        model_diastole.save_weights('weights_diastole.hdf5', overwrite=True)

        # for best (lowest) val losses, save weights
        if val_loss_systole < min_val_loss_systole:
            min_val_loss_systole = val_loss_systole
            model_systole.save_weights('weights_systole_best.hdf5', overwrite=True)

        if val_loss_diastole < min_val_loss_diastole:
            min_val_loss_diastole = val_loss_diastole
            model_diastole.save_weights('weights_diastole_best.hdf5', overwrite=True)

        # save best (lowest) val losses in file (to be later used for generating submission)
        with open('val_loss.txt', mode='w+') as f:
            f.write(str(min_val_loss_systole))
            f.write('\n')
            f.write(str(min_val_loss_diastole))
    print('best case accuracy train: {0}, test: {1}'.format(crps_train_min, crps_test_min))
    return crps_test_min  

optimal_parameters, details, _ = optunity.minimize(train_accuracy, num_evals=20,
                                                   drop_out_1=[0.001, 1.0], drop_out_2=[0.001, 1.0], drop_out_3=[0.001, 1.0], drop_out_4=[0.001, 1.0])

print ('found optimal parameters: {0}'.format(optimal_parameters))
np.savetxt("best_acc_hyper_param.txt", optimal_parameters , delimiter=",")
#print(train_accuracy(0.25, 0.25, 0.25, 0.5))