def shuffle(x, y, steps=10, slc=100000, plot=True): scores_train, scores_test = [], [] best_clf, best_score = None, None z = x.copy(deep=True) z["ReturnQuantityBin"] = y.ReturnQuantityBin for k in range(steps): u = z.sample(frac=1) v = u.loc[:, ["ReturnQuantityBin"]] u = u.drop(["ReturnQuantityBin"], axis=1) #clf = LogisticRegression() clf = MLPRegressor(solver='sgd', alpha=1e-5, hidden_layer_sizes=(100, 100, 100, 100), random_state=1) clf.fit(u.iloc[:slc], v.ReturnQuantityBin[:slc]) predict_train = clf.predict_proba(u.iloc[:slc]) score_train = roc_auc_score(v.ReturnQuantityBin[:slc], predict_train[:, 1]) predict_test = clf.predict_proba(u.iloc[slc:2 * slc]) score_test = roc_auc_score(v.ReturnQuantityBin[slc:2 * slc], predict_test[:, 1]) if best_clf is None or score_test > best_score: best_clf, best_score = clf, score_test if plot: print "test", k, "\ttrain:", score_train, "\ttest:", score_test scores_train.append(score_train) scores_test.append(score_test) if plot: plt.figure(figsize=(16, 10)) plt.xlabel("train score") plt.ylabel("test score") plt.plot(scores_train, scores_test, '+') plt.show() return scores_train, scores_test, best_clf, best_score
class MLP(IterativeComponentWithSampleWeight, BaseRegressionModel): def __init__(self, activation, solver, alpha, tol, hidden_size, learning_rate='constant', learning_rate_init=0.001, power_t=0.5, momentum=0.9, nesterovs_momentum=True, beta1=0.9, random_state=None): self.activation = activation self.solver = solver self.alpha = alpha self.learning_rate = learning_rate self.learning_rate_init = learning_rate_init self.tol = tol self.hidden_size = hidden_size self.momentum = momentum self.nesterovs_momentum = nesterovs_momentum self.beta1 = beta1 self.power_t = power_t self.random_state = random_state self.estimator = None self.fully_fit_ = False self.time_limit = None self.start_time = time.time() def iterative_fit(self, X, y, n_iter=2, refit=False, sample_weight=None): from sklearn.neural_network import MLPRegressor # Need to fit at least two iterations, otherwise early stopping will not # work because we cannot determine whether the algorithm actually # converged. The only way of finding this out is if the sgd spends less # iterations than max_iter. If max_iter == 1, it has to spend at least # one iteration and will always spend at least one iteration, so we # cannot know about convergence. if isinstance(self.solver, tuple): nested_solver = self.solver self.solver = nested_solver[0] if self.solver == 'sgd': self.momentum = nested_solver[1]['momentum'] self.nesterovs_momentum = nested_solver[1]['nesterovs_momentum'] nested_learning_rate = nested_solver[1]['learning_rate'] self.learning_rate = nested_learning_rate[0] if self.learning_rate == 'invscaling': self.power_t = nested_learning_rate[1]['power_t'] elif self.solver == 'adam': self.beta1 = nested_solver[1]['beta1'] if refit: self.estimator = None if self.estimator is None: self.fully_fit_ = False if self.learning_rate is None: self.learning_rate = "constant" self.alpha = float(self.alpha) self.power_t = float(self.power_t) if self.power_t is not None \ else 0.5 self.tol = float(self.tol) self.estimator = MLPRegressor(hidden_layer_sizes=(self.hidden_size, self.hidden_size), activation=self.activation, solver=self.solver, alpha=self.alpha, learning_rate=self.learning_rate, learning_rate_init=self.learning_rate_init, power_t=self.power_t, max_iter=n_iter, shuffle=True, tol=self.tol, warm_start=True, momentum=self.momentum, n_iter_no_change=50, nesterovs_momentum=self.nesterovs_momentum, beta_1=self.beta1) self.estimator.fit(X, y) else: self.estimator.max_iter += n_iter self.estimator.max_iter = min(self.estimator.max_iter, 2048) for i in range(n_iter): self.estimator.fit(X, y) if self.estimator._no_improvement_count > self.estimator.n_iter_no_change: self.fully_fit_ = True break if self.estimator.max_iter >= 2048: self.fully_fit_ = True return self def configuration_fully_fitted(self): if self.estimator is None: return False elif not hasattr(self, 'fully_fit_'): return False else: return self.fully_fit_ def predict(self, X): if self.estimator is None: raise NotImplementedError() return self.estimator.predict(X) def predict_proba(self, X): if self.estimator is None: raise NotImplementedError() return self.estimator.predict_proba(X) @staticmethod def get_properties(dataset_properties=None): return {'shortname': 'MLP Regressor', 'name': 'Multi-layer Perceptron Regressor', 'handles_regression': True, 'handles_classification': False, 'handles_multiclass': False, 'handles_multilabel': False, 'is_deterministic': True, 'input': (DENSE, SPARSE, UNSIGNED_DATA), 'output': (PREDICTIONS,)} @staticmethod def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() hidden_size = UniformIntegerHyperparameter("hidden_size", 100, 500, default_value=200) activation = CategoricalHyperparameter("activation", ["identity", "logistic", "tanh", "relu"], default_value="relu") solver = CategoricalHyperparameter("solver", ["sgd", "adam"], default_value="adam") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1., log=True, default_value=0.0001) learning_rate = CategoricalHyperparameter( "learning_rate", ["adaptive", "invscaling", "constant"], default_value="constant") learning_rate_init = UniformFloatHyperparameter( "learning_rate_init", 1e-4, 3e-1, default_value=0.001, log=True) tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True, default_value=1e-4) momentum = UniformFloatHyperparameter("momentum", 0.6, 1, q=0.05, default_value=0.9) nesterovs_momentum = CategoricalHyperparameter("nesterovs_momentum", [True, False], default_value=True) beta1 = UniformFloatHyperparameter("beta1", 0.6, 1, default_value=0.9) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True, default_value=0.5) cs.add_hyperparameters( [hidden_size, activation, solver, alpha, learning_rate, learning_rate_init, tol, momentum, nesterovs_momentum, beta1, power_t]) learning_rate_condition = EqualsCondition(learning_rate, solver, "sgd") momentum_condition = EqualsCondition(momentum, solver, "sgd") nesterovs_momentum_condition = EqualsCondition(nesterovs_momentum, solver, "sgd") beta1_condition = EqualsCondition(beta1, solver, "adam") power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([learning_rate_condition, momentum_condition, nesterovs_momentum_condition, beta1_condition, power_t_condition]) return cs elif optimizer == 'tpe': space = {'hidden_size': hp.randint("mlp_hidden_size", 400) + 100, 'activation': hp.choice('mlp_activation', ["identity", "logistic", "tanh", "relu"]), 'solver': hp.choice('mlp_solver', [("sgd", {'learning_rate': hp.choice('mlp_learning_rate', [("adaptive", {}), ("constant", {}), ("invscaling", { 'power_t': hp.uniform('mlp_power_t', 1e-5, 1)})]), 'momentum': hp.uniform('mlp_momentum', 0.6, 1), 'nesterovs_momentum': hp.choice('mlp_nesterovs_momentum', [True, False])}), ("adam", {'beta1': hp.uniform('mlp_beta1', 0.6, 1)})]), 'alpha': hp.loguniform('mlp_alpha', np.log(1e-7), np.log(1e-1)), 'learning_rate_init': hp.loguniform('mlp_learning_rate_init', np.log(1e-6), np.log(1e-1)), 'tol': hp.loguniform('mlp_tol', np.log(1e-5), np.log(1e-1))} return space