Exemplo n.º 1
0
def shuffle(x, y, steps=10, slc=100000, plot=True):
    scores_train, scores_test = [], []
    best_clf, best_score = None, None

    z = x.copy(deep=True)
    z["ReturnQuantityBin"] = y.ReturnQuantityBin

    for k in range(steps):
        u = z.sample(frac=1)
        v = u.loc[:, ["ReturnQuantityBin"]]
        u = u.drop(["ReturnQuantityBin"], axis=1)

        #clf = LogisticRegression()
        clf = MLPRegressor(solver='sgd',
                           alpha=1e-5,
                           hidden_layer_sizes=(100, 100, 100, 100),
                           random_state=1)
        clf.fit(u.iloc[:slc], v.ReturnQuantityBin[:slc])

        predict_train = clf.predict_proba(u.iloc[:slc])
        score_train = roc_auc_score(v.ReturnQuantityBin[:slc],
                                    predict_train[:, 1])

        predict_test = clf.predict_proba(u.iloc[slc:2 * slc])
        score_test = roc_auc_score(v.ReturnQuantityBin[slc:2 * slc],
                                   predict_test[:, 1])

        if best_clf is None or score_test > best_score:
            best_clf, best_score = clf, score_test

        if plot:
            print "test", k, "\ttrain:", score_train, "\ttest:", score_test

        scores_train.append(score_train)
        scores_test.append(score_test)

    if plot:
        plt.figure(figsize=(16, 10))
        plt.xlabel("train score")
        plt.ylabel("test score")
        plt.plot(scores_train, scores_test, '+')
        plt.show()

    return scores_train, scores_test, best_clf, best_score
Exemplo n.º 2
0
class MLP(IterativeComponentWithSampleWeight, BaseRegressionModel):
    def __init__(self, activation, solver, alpha, tol, hidden_size, learning_rate='constant', learning_rate_init=0.001,
                 power_t=0.5, momentum=0.9, nesterovs_momentum=True,
                 beta1=0.9, random_state=None):
        self.activation = activation
        self.solver = solver
        self.alpha = alpha
        self.learning_rate = learning_rate
        self.learning_rate_init = learning_rate_init
        self.tol = tol
        self.hidden_size = hidden_size
        self.momentum = momentum
        self.nesterovs_momentum = nesterovs_momentum
        self.beta1 = beta1
        self.power_t = power_t
        self.random_state = random_state
        self.estimator = None
        self.fully_fit_ = False
        self.time_limit = None
        self.start_time = time.time()

    def iterative_fit(self, X, y, n_iter=2, refit=False, sample_weight=None):
        from sklearn.neural_network import MLPRegressor

        # Need to fit at least two iterations, otherwise early stopping will not
        # work because we cannot determine whether the algorithm actually
        # converged. The only way of finding this out is if the sgd spends less
        # iterations than max_iter. If max_iter == 1, it has to spend at least
        # one iteration and will always spend at least one iteration, so we
        # cannot know about convergence.

        if isinstance(self.solver, tuple):
            nested_solver = self.solver
            self.solver = nested_solver[0]
            if self.solver == 'sgd':
                self.momentum = nested_solver[1]['momentum']
                self.nesterovs_momentum = nested_solver[1]['nesterovs_momentum']
                nested_learning_rate = nested_solver[1]['learning_rate']
                self.learning_rate = nested_learning_rate[0]
                if self.learning_rate == 'invscaling':
                    self.power_t = nested_learning_rate[1]['power_t']
            elif self.solver == 'adam':
                self.beta1 = nested_solver[1]['beta1']

        if refit:
            self.estimator = None

        if self.estimator is None:
            self.fully_fit_ = False
            if self.learning_rate is None:
                self.learning_rate = "constant"
            self.alpha = float(self.alpha)
            self.power_t = float(self.power_t) if self.power_t is not None \
                else 0.5
            self.tol = float(self.tol)

            self.estimator = MLPRegressor(hidden_layer_sizes=(self.hidden_size, self.hidden_size),
                                          activation=self.activation,
                                          solver=self.solver,
                                          alpha=self.alpha,
                                          learning_rate=self.learning_rate,
                                          learning_rate_init=self.learning_rate_init,
                                          power_t=self.power_t,
                                          max_iter=n_iter,
                                          shuffle=True,
                                          tol=self.tol,
                                          warm_start=True,
                                          momentum=self.momentum,
                                          n_iter_no_change=50,
                                          nesterovs_momentum=self.nesterovs_momentum,
                                          beta_1=self.beta1)
            self.estimator.fit(X, y)
        else:
            self.estimator.max_iter += n_iter
            self.estimator.max_iter = min(self.estimator.max_iter, 2048)
            for i in range(n_iter):
                self.estimator.fit(X, y)
                if self.estimator._no_improvement_count > self.estimator.n_iter_no_change:
                    self.fully_fit_ = True
                    break

        if self.estimator.max_iter >= 2048:
            self.fully_fit_ = True

        return self

    def configuration_fully_fitted(self):
        if self.estimator is None:
            return False
        elif not hasattr(self, 'fully_fit_'):
            return False
        else:
            return self.fully_fit_

    def predict(self, X):
        if self.estimator is None:
            raise NotImplementedError()
        return self.estimator.predict(X)

    def predict_proba(self, X):
        if self.estimator is None:
            raise NotImplementedError()
        return self.estimator.predict_proba(X)

    @staticmethod
    def get_properties(dataset_properties=None):
        return {'shortname': 'MLP Regressor',
                'name': 'Multi-layer Perceptron Regressor',
                'handles_regression': True,
                'handles_classification': False,
                'handles_multiclass': False,
                'handles_multilabel': False,
                'is_deterministic': True,
                'input': (DENSE, SPARSE, UNSIGNED_DATA),
                'output': (PREDICTIONS,)}

    @staticmethod
    def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
        if optimizer == 'smac':
            cs = ConfigurationSpace()

            hidden_size = UniformIntegerHyperparameter("hidden_size", 100, 500, default_value=200)
            activation = CategoricalHyperparameter("activation", ["identity", "logistic", "tanh", "relu"],
                                                   default_value="relu")
            solver = CategoricalHyperparameter("solver", ["sgd", "adam"], default_value="adam")

            alpha = UniformFloatHyperparameter(
                "alpha", 1e-7, 1., log=True, default_value=0.0001)

            learning_rate = CategoricalHyperparameter(
                "learning_rate", ["adaptive", "invscaling", "constant"],
                default_value="constant")

            learning_rate_init = UniformFloatHyperparameter(
                "learning_rate_init", 1e-4, 3e-1, default_value=0.001, log=True)

            tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True,
                                             default_value=1e-4)
            momentum = UniformFloatHyperparameter("momentum", 0.6, 1, q=0.05, default_value=0.9)

            nesterovs_momentum = CategoricalHyperparameter("nesterovs_momentum", [True, False], default_value=True)
            beta1 = UniformFloatHyperparameter("beta1", 0.6, 1, default_value=0.9)
            power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True,
                                                 default_value=0.5)
            cs.add_hyperparameters(
                [hidden_size, activation, solver, alpha, learning_rate, learning_rate_init, tol, momentum,
                 nesterovs_momentum, beta1,
                 power_t])

            learning_rate_condition = EqualsCondition(learning_rate, solver, "sgd")
            momentum_condition = EqualsCondition(momentum, solver, "sgd")
            nesterovs_momentum_condition = EqualsCondition(nesterovs_momentum, solver, "sgd")
            beta1_condition = EqualsCondition(beta1, solver, "adam")

            power_t_condition = EqualsCondition(power_t, learning_rate,
                                                "invscaling")

            cs.add_conditions([learning_rate_condition, momentum_condition,
                               nesterovs_momentum_condition, beta1_condition, power_t_condition])

            return cs
        elif optimizer == 'tpe':
            space = {'hidden_size': hp.randint("mlp_hidden_size", 400) + 100,
                     'activation': hp.choice('mlp_activation', ["identity", "logistic", "tanh", "relu"]),
                     'solver': hp.choice('mlp_solver',
                                         [("sgd", {'learning_rate': hp.choice('mlp_learning_rate',
                                                                              [("adaptive", {}),
                                                                               ("constant", {}),
                                                                               ("invscaling", {
                                                                                   'power_t': hp.uniform('mlp_power_t',
                                                                                                         1e-5, 1)})]),
                                                   'momentum': hp.uniform('mlp_momentum', 0.6, 1),
                                                   'nesterovs_momentum': hp.choice('mlp_nesterovs_momentum',
                                                                                   [True, False])}),
                                          ("adam", {'beta1': hp.uniform('mlp_beta1', 0.6, 1)})]),
                     'alpha': hp.loguniform('mlp_alpha', np.log(1e-7), np.log(1e-1)),
                     'learning_rate_init': hp.loguniform('mlp_learning_rate_init', np.log(1e-6), np.log(1e-1)),
                     'tol': hp.loguniform('mlp_tol', np.log(1e-5), np.log(1e-1))}

            return space