def iterative_fit(self, X, y, n_iter=1, refit=False): from sklearn.linear_model.stochastic_gradient import SGDRegressor import sklearn.preprocessing if refit: self.estimator = None self.scaler = None if self.estimator is None: self._iterations = 0 self.alpha = float(self.alpha) self.fit_intercept = self.fit_intercept == "True" self.n_iter = int(self.n_iter) self.l1_ratio = float(self.l1_ratio) if self.l1_ratio is not None else 0.15 self.epsilon = float(self.epsilon) if self.epsilon is not None else 0.1 self.eta0 = float(self.eta0) self.power_t = float(self.power_t) if self.power_t is not None else 0.25 self.average = self.average == "True" self.estimator = SGDRegressor( loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, n_iter=self.n_iter, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=self.epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state, ) self.scaler = sklearn.preprocessing.StandardScaler(copy=True) self.scaler.fit(y) Y_scaled = self.scaler.transform(y) self.estimator.n_iter = n_iter self._iterations += n_iter print(n_iter) self.estimator.partial_fit(X, Y_scaled) return self
def iterative_fit(self, X, y, n_iter=1, refit=False): if refit: self.estimator = None self.scaler = None if self.estimator is None: self.alpha = float(self.alpha) self.fit_intercept = self.fit_intercept == 'True' self.n_iter = int(self.n_iter) self.l1_ratio = float( self.l1_ratio) if self.l1_ratio is not None else 0.15 self.epsilon = float( self.epsilon) if self.epsilon is not None else 0.1 self.eta0 = float(self.eta0) self.power_t = float( self.power_t) if self.power_t is not None else 0.25 self.average = self.average == 'True' self.estimator = SGDRegressor(loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, n_iter=self.n_iter, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=self.epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state) self.scaler = sklearn.preprocessing.StandardScaler(copy=True) self.scaler.fit(y) Y_scaled = self.scaler.transform(y) self.estimator.n_iter += n_iter self.estimator.fit(X, Y_scaled) return self
def iterative_fit(self, X, y, n_iter=2, refit=False, sample_weight=None): from sklearn.linear_model.stochastic_gradient import SGDRegressor # Need to fit at least two iterations, otherwise early stopping will not # work because we cannot determine whether the algorithm actually # converged. The only way of finding this out is if the sgd spends less # iterations than max_iter. If max_iter == 1, it has to spend at least # one iteration and will always spend at least one iteration, so we # cannot know about convergence. if refit: self.estimator = None if self.estimator is None: self.fully_fit_ = False self.alpha = float(self.alpha) if not check_none(self.epsilon_insensitive): self.epsilon_insensitive = float(self.epsilon_insensitive) self.l1_ratio = float(self.l1_ratio) if self.l1_ratio is not None \ else 0.15 self.epsilon_huber = float(self.epsilon_huber) if self.epsilon_huber is not None \ else 0.1 self.eta0 = float(self.eta0) if self.eta0 is not None else 0.01 self.power_t = float(self.power_t) if self.power_t is not None \ else 0.5 self.average = check_for_bool(self.average) self.fit_intercept = check_for_bool(self.fit_intercept) self.tol = float(self.tol) if self.loss == "huber": epsilon = self.epsilon_huber elif self.loss in [ "epsilon_insensitive", "squared_epsilon_insensitive" ]: epsilon = self.epsilon_insensitive else: epsilon = None self.estimator = SGDRegressor(loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, max_iter=n_iter, tol=self.tol, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state, warm_start=True) self.estimator.fit(X, y, sample_weight=sample_weight) else: self.estimator.max_iter += n_iter self.estimator.max_iter = min(self.estimator.max_iter, 512) self.estimator._validate_params() self.estimator._partial_fit( X, y, alpha=self.estimator.alpha, C=1.0, loss=self.estimator.loss, learning_rate=self.estimator.learning_rate, max_iter=n_iter, sample_weight=sample_weight, coef_init=None, intercept_init=None) if self.estimator.max_iter >= 512 or n_iter > self.estimator.n_iter_: self.fully_fit_ = True return self
class SGD( IterativeComponentWithSampleWeight, BaseRegressionModel, ): def __init__(self, loss, penalty, alpha, fit_intercept, tol, learning_rate, epsilon_insensitive, l1_ratio=0.15, epsilon_huber=0.1, eta0=0.01, power_t=0.5, average=False, random_state=None): self.loss = loss self.penalty = penalty self.alpha = alpha self.fit_intercept = fit_intercept self.tol = tol self.learning_rate = learning_rate self.l1_ratio = l1_ratio self.epsilon_huber = epsilon_huber self.epsilon_insensitive = epsilon_insensitive self.eta0 = eta0 self.power_t = power_t self.random_state = random_state self.average = average self.estimator = None self.start_time = time.time() self.time_limit = None def iterative_fit(self, X, y, n_iter=2, refit=False, sample_weight=None): from sklearn.linear_model.stochastic_gradient import SGDRegressor # Need to fit at least two iterations, otherwise early stopping will not # work because we cannot determine whether the algorithm actually # converged. The only way of finding this out is if the sgd spends less # iterations than max_iter. If max_iter == 1, it has to spend at least # one iteration and will always spend at least one iteration, so we # cannot know about convergence. if refit: self.estimator = None if self.estimator is None: self.fully_fit_ = False self.alpha = float(self.alpha) if not check_none(self.epsilon_insensitive): self.epsilon_insensitive = float(self.epsilon_insensitive) self.l1_ratio = float(self.l1_ratio) if self.l1_ratio is not None \ else 0.15 self.epsilon_huber = float(self.epsilon_huber) if self.epsilon_huber is not None \ else 0.1 self.eta0 = float(self.eta0) if self.eta0 is not None else 0.01 self.power_t = float(self.power_t) if self.power_t is not None \ else 0.5 self.average = check_for_bool(self.average) self.fit_intercept = check_for_bool(self.fit_intercept) self.tol = float(self.tol) if self.loss == "huber": epsilon = self.epsilon_huber elif self.loss in [ "epsilon_insensitive", "squared_epsilon_insensitive" ]: epsilon = self.epsilon_insensitive else: epsilon = None self.estimator = SGDRegressor(loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, max_iter=n_iter, tol=self.tol, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state, warm_start=True) self.estimator.fit(X, y, sample_weight=sample_weight) else: self.estimator.max_iter += n_iter self.estimator.max_iter = min(self.estimator.max_iter, 512) self.estimator._validate_params() self.estimator._partial_fit( X, y, alpha=self.estimator.alpha, C=1.0, loss=self.estimator.loss, learning_rate=self.estimator.learning_rate, max_iter=n_iter, sample_weight=sample_weight, coef_init=None, intercept_init=None) if self.estimator.max_iter >= 512 or n_iter > self.estimator.n_iter_: self.fully_fit_ = True return self def configuration_fully_fitted(self): if self.estimator is None: return False elif not hasattr(self, 'fully_fit_'): return False else: return self.fully_fit_ def predict(self, X): if self.estimator is None: raise NotImplementedError() return self.estimator.predict(X) @staticmethod def get_properties(dataset_properties=None): return { 'shortname': 'SGD Regressor', 'name': 'Stochastic Gradient Descent Regressor', 'handles_regression': True, 'handles_classification': False, 'handles_multiclass': False, 'handles_multilabel': False, 'is_deterministic': True, 'input': (DENSE, SPARSE, UNSIGNED_DATA), 'output': (PREDICTIONS, ) } @staticmethod def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", [ "squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive" ], default_value="squared_loss") penalty = CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter("alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter("l1_ratio", 1e-9, 1, log=True, default_value=0.15) fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True, default_value=1e-4) epsilon_huber = UniformFloatHyperparameter("epsilon_huber", 1e-5, 1e-1, default_value=1e-4, log=True) epsilon_insensitive = UniformFloatHyperparameter("epsilon_insensitive", 1e-5, 1e-1, default_value=1e-4, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter("eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True, default_value=0.5) average = CategoricalHyperparameter("average", ["False", "True"], default_value="False") cs.add_hyperparameters([ loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon_huber, epsilon_insensitive, learning_rate, eta0, power_t, average ]) elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_huber_condition = EqualsCondition(epsilon_huber, loss, "huber") epsilon_insensitive_condition = InCondition( epsilon_insensitive, loss, ["epsilon_insensitive", "squared_epsilon_insensitive"]) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) cs.add_conditions([ elasticnet, epsilon_huber_condition, epsilon_insensitive_condition, power_t_condition, eta0_in_inv_con ]) return cs
plt.show() ############################################################################### # main code start_time = time.time() # benchmark bulk/atomic prediction speed for various regressors configuration = { 'n_train': int(1e3), 'n_test': int(1e2), 'n_features': int(1e2), 'estimators': [ {'name': 'Linear Model', 'instance': SGDRegressor(penalty='elasticnet', alpha=0.01, l1_ratio=0.25, fit_intercept=True), 'complexity_label': 'non-zero coefficients', 'complexity_computer': lambda clf: np.count_nonzero(clf.coef_)}, {'name': 'RandomForest', 'instance': RandomForestRegressor(), 'complexity_label': 'estimators', 'complexity_computer': lambda clf: clf.n_estimators}, {'name': 'SVR', 'instance': SVR(kernel='rbf'), 'complexity_label': 'support vectors', 'complexity_computer': lambda clf: len(clf.support_vectors_)}, ] } benchmark(configuration) # benchmark n_features influence on prediction speed
x_scaler = StandardScaler().fit(x_train) y_scaler = StandardScaler().fit(y_train.values.reshape(-1, 1)) #transform: 执行数据标准化 #测试数据和预测数据的标准化的方式要和训练数据标准化的方式一样, 必须用同一个scaler来进行transform #数据标准化公式,每个数据均是 (x - 平均值)/标准差 standardized_x_train = x_scaler.transform(x_train) standardized_y_train = y_scaler.transform(y_train.values.reshape(-1, 1)).ravel() standardized_x_test = x_scaler.transform(x_test) standardized_y_test = y_scaler.transform(y_test.values.reshape(-1, 1)).ravel() #loss:要是用的损失函数,默认是squared_loss,方差拟合 #penalty:使用的惩罚 lm = SGDRegressor(loss='squared_loss', penalty='none', max_iter=args.num_epochs) #使用梯度下降模型 lm.fit(X=standardized_x_train, y=standardized_y_train) #predict:进行数据预测 #scaler.var_是方差,np.sqrt(y_scaler.var_)就是标准差,y_scaler.scale_也是标准差,是一样的 #实际输出结果就是 (模拟的输出结果 * 标准差) + 平均数 和标准化过程刚好相反 pred_train = (lm.predict(standardized_x_train) * y_scaler.scale_) + y_scaler.mean_ pred_test = (lm.predict(standardized_x_test) * np.sqrt(y_scaler.var_)) + y_scaler.mean_ #测试我们自己的数据 X_infer = np.array((0, 1, 2), dtype=np.float32) standardized_X_infer = x_scaler.transform(X_infer.reshape(-1, 1)) pred_infer = (lm.predict(standardized_X_infer) *
class SGD(AutoSklearnRegressionAlgorithm): def __init__( self, loss, penalty, alpha, fit_intercept, n_iter, learning_rate, l1_ratio=0.15, epsilon=0.1, eta0=0.01, power_t=0.5, average=False, random_state=None, ): self.loss = loss self.penalty = penalty self.alpha = alpha self.fit_intercept = fit_intercept self.n_iter = n_iter self.learning_rate = learning_rate self.l1_ratio = l1_ratio self.epsilon = epsilon self.eta0 = eta0 self.power_t = power_t self.random_state = random_state self.average = average self.estimator = None self.scaler = None def fit(self, X, y): while not self.configuration_fully_fitted(): self.iterative_fit(X, y, n_iter=1) return self def iterative_fit(self, X, y, n_iter=1, refit=False): from sklearn.linear_model.stochastic_gradient import SGDRegressor import sklearn.preprocessing if refit: self.estimator = None self.scaler = None if self.estimator is None: self._iterations = 0 self.alpha = float(self.alpha) self.fit_intercept = self.fit_intercept == "True" self.n_iter = int(self.n_iter) self.l1_ratio = float(self.l1_ratio) if self.l1_ratio is not None else 0.15 self.epsilon = float(self.epsilon) if self.epsilon is not None else 0.1 self.eta0 = float(self.eta0) self.power_t = float(self.power_t) if self.power_t is not None else 0.25 self.average = self.average == "True" self.estimator = SGDRegressor( loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, n_iter=self.n_iter, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=self.epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state, ) self.scaler = sklearn.preprocessing.StandardScaler(copy=True) self.scaler.fit(y) Y_scaled = self.scaler.transform(y) self.estimator.n_iter = n_iter self._iterations += n_iter print(n_iter) self.estimator.partial_fit(X, Y_scaled) return self def configuration_fully_fitted(self): if self.estimator is None: return False return not self._iterations < self.n_iter def predict(self, X): if self.estimator is None: raise NotImplementedError() Y_pred = self.estimator.predict(X) return self.scaler.inverse_transform(Y_pred) @staticmethod def get_properties(dataset_properties=None): return { "shortname": "SGD Regressor", "name": "Stochastic Gradient Descent Regressor", "handles_missing_values": False, "handles_nominal_values": False, "handles_numerical_features": True, "prefers_data_scaled": True, "prefers_data_normalized": True, "handles_regression": True, "handles_classification": False, "handles_multiclass": False, "handles_multilabel": False, "is_deterministic": True, "handles_sparse": True, "input": (DENSE, SPARSE, UNSIGNED_DATA), "output": (PREDICTIONS,), # TODO find out what is best used here! "preferred_dtype": None, } @staticmethod def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = cs.add_hyperparameter( CategoricalHyperparameter( "loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default="squared_loss", ) ) penalty = cs.add_hyperparameter(CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet"], default="l2")) alpha = cs.add_hyperparameter(UniformFloatHyperparameter("alpha", 10e-7, 1e-1, log=True, default=0.01)) l1_ratio = cs.add_hyperparameter(UniformFloatHyperparameter("l1_ratio", 1e-9, 1.0, log=True, default=0.15)) fit_intercept = cs.add_hyperparameter(UnParametrizedHyperparameter("fit_intercept", "True")) n_iter = cs.add_hyperparameter(UniformIntegerHyperparameter("n_iter", 5, 1000, log=True, default=20)) epsilon = cs.add_hyperparameter(UniformFloatHyperparameter("epsilon", 1e-5, 1e-1, default=1e-4, log=True)) learning_rate = cs.add_hyperparameter( CategoricalHyperparameter("learning_rate", ["optimal", "invscaling", "constant"], default="optimal") ) eta0 = cs.add_hyperparameter(UniformFloatHyperparameter("eta0", 10 ** -7, 0.1, default=0.01)) power_t = cs.add_hyperparameter(UniformFloatHyperparameter("power_t", 1e-5, 1, default=0.5)) average = cs.add_hyperparameter(CategoricalHyperparameter("average", ["False", "True"], default="False")) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition(epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 seems to be always active according to the source code; when # learning_rate is set to optimial, eta0 is the starting value: # https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx # eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling") # eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant") # eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_condition(elasticnet) cs.add_condition(epsilon_condition) cs.add_condition(power_t_condition) return cs
np.random.shuffle(inds) coef[inds[n_features / 2:]] = 0 # sparsify coef print("true coef sparsity: %f" % sparsity_ratio(coef)) y = np.dot(X, coef) # add noise y += 0.01 * np.random.normal((n_samples, )) # Split data in train set and test set n_samples = X.shape[0] X_train, y_train = X[:n_samples / 2], y[:n_samples / 2] X_test, y_test = X[n_samples / 2:], y[n_samples / 2:] print("test data sparsity: %f" % sparsity_ratio(X_test)) ############################################################################### clf = SGDRegressor(penalty='l1', alpha=.2, fit_intercept=True, n_iter=2000) clf.fit(X_train, y_train) print("model sparsity: %f" % sparsity_ratio(clf.coef_)) @profile def benchmark_dense_predict(): for _ in range(300): clf.predict(X_test) @profile def benchmark_sparse_predict(): X_test_sparse = csr_matrix(X_test) for _ in range(300): clf.predict(X_test_sparse)
# Benchmark bulk/atomic prediction speed for various regressors configuration = { 'n_train': int(1e3), 'n_test': int(1e2), 'n_features': int(1e2), 'estimators': [ { 'name': 'Linear Model', 'instance': SGDRegressor(penalty='elasticnet', alpha=0.01, l1_ratio=0.25, fit_intercept=True, tol=1e-4), 'complexity_label': 'non-zero coefficients', 'complexity_computer': lambda clf: np.count_nonzero(clf.coef_) }, { 'name': 'RandomForest', 'instance': RandomForestRegressor(), 'complexity_label': 'estimators', 'complexity_computer': lambda clf: clf.n_estimators }, { 'name': 'SVR',
start_time = time.time() configuration = { 'n_train': int(1e3), 'n_test': int(1e2), 'n_features': int(1e2), 'estimators': [{ 'name': "Linear Model", 'instance': SGDRegressor(penalty='l2', alpha=0.1, tol=1e-4), 'complexity_label': 'non-zero coefficients', 'complexity_computer': lambda clf: np.count_nonzero(clf.coef_) }, { 'name': 'RandomForest', 'instance': RandomForestRegressor(), 'complexity_label': 'estimator', 'complexity_computer': lambda clf: clf.n_estimators }, { 'name': 'SVR', 'instance': SVR(kernel='rbf'), 'complexity_label': 'support vectors', 'complexity_computer': lambda clf: len(clf.support_vectors_) }]
def iterative_fit(self, X, y, n_iter=2, refit=False): from sklearn.linear_model.stochastic_gradient import SGDRegressor import sklearn.preprocessing # Need to fit at least two iterations, otherwise early stopping will not # work because we cannot determine whether the algorithm actually # converged. The only way of finding this out is if the sgd spends less # iterations than max_iter. If max_iter == 1, it has to spend at least # one iteration and will always spend at least one iteration, so we # cannot know about convergence. n_iter = max(n_iter, 2) if refit: self.estimator = None self.scaler = None if self.estimator is None: self.alpha = float(self.alpha) self.fit_intercept = check_for_bool(self.fit_intercept) self.tol = float(self.tol) self.l1_ratio = float( self.l1_ratio) if self.l1_ratio is not None else 0.15 self.epsilon = float( self.epsilon) if self.epsilon is not None else 0.1 self.eta0 = float(self.eta0) self.power_t = float( self.power_t) if self.power_t is not None else 0.25 self.average = check_for_bool(self.average) self.estimator = SGDRegressor(loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, max_iter=n_iter, tol=self.tol, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=self.epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state, warm_start=True) self.scaler = sklearn.preprocessing.StandardScaler(copy=True) self.scaler.fit(y.reshape((-1, 1))) Y_scaled = self.scaler.transform(y.reshape((-1, 1))).ravel() self.estimator.fit(X, Y_scaled) else: self.estimator.max_iter += n_iter self.estimator.max_iter = min(self.estimator.max_iter, 512) Y_scaled = self.scaler.transform(y.reshape((-1, 1))).ravel() self.estimator._validate_params() self.estimator._partial_fit( X, Y_scaled, alpha=self.estimator.alpha, C=1.0, loss=self.estimator.loss, learning_rate=self.estimator.learning_rate, max_iter=n_iter, sample_weight=None, coef_init=None, intercept_init=None ) if self.estimator.max_iter >= 512 or n_iter > self.estimator.n_iter_: self.fully_fit_ = True return self
class SGD( IterativeComponent, AutoSklearnRegressionAlgorithm, ): def __init__(self, loss, penalty, alpha, fit_intercept, tol, learning_rate, l1_ratio=0.15, epsilon=0.1, eta0=0.01, power_t=0.5, average=False, random_state=None): self.loss = loss self.penalty = penalty self.alpha = alpha self.fit_intercept = fit_intercept self.tol = tol self.learning_rate = learning_rate self.l1_ratio = l1_ratio self.epsilon = epsilon self.eta0 = eta0 self.power_t = power_t self.random_state = random_state self.average = average self.estimator = None self.scaler = None def iterative_fit(self, X, y, n_iter=2, refit=False): from sklearn.linear_model.stochastic_gradient import SGDRegressor import sklearn.preprocessing # Need to fit at least two iterations, otherwise early stopping will not # work because we cannot determine whether the algorithm actually # converged. The only way of finding this out is if the sgd spends less # iterations than max_iter. If max_iter == 1, it has to spend at least # one iteration and will always spend at least one iteration, so we # cannot know about convergence. n_iter = max(n_iter, 2) if refit: self.estimator = None self.scaler = None if self.estimator is None: self.alpha = float(self.alpha) self.fit_intercept = check_for_bool(self.fit_intercept) self.tol = float(self.tol) self.l1_ratio = float( self.l1_ratio) if self.l1_ratio is not None else 0.15 self.epsilon = float( self.epsilon) if self.epsilon is not None else 0.1 self.eta0 = float(self.eta0) self.power_t = float( self.power_t) if self.power_t is not None else 0.25 self.average = check_for_bool(self.average) self.estimator = SGDRegressor(loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, max_iter=n_iter, tol=self.tol, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=self.epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state, warm_start=True) self.scaler = sklearn.preprocessing.StandardScaler(copy=True) self.scaler.fit(y.reshape((-1, 1))) Y_scaled = self.scaler.transform(y.reshape((-1, 1))).ravel() self.estimator.fit(X, Y_scaled) else: self.estimator.max_iter += n_iter self.estimator.max_iter = min(self.estimator.max_iter, 512) Y_scaled = self.scaler.transform(y.reshape((-1, 1))).ravel() self.estimator._validate_params() self.estimator._partial_fit( X, Y_scaled, alpha=self.estimator.alpha, C=1.0, loss=self.estimator.loss, learning_rate=self.estimator.learning_rate, max_iter=n_iter, sample_weight=None, coef_init=None, intercept_init=None ) if self.estimator.max_iter >= 512 or n_iter > self.estimator.n_iter_: self.fully_fit_ = True return self def configuration_fully_fitted(self): if self.estimator is None: return False elif not hasattr(self, 'fully_fit_'): return False else: return self.fully_fit_ def predict(self, X): if self.estimator is None: raise NotImplementedError() Y_pred = self.estimator.predict(X) return self.scaler.inverse_transform(Y_pred) @staticmethod def get_properties(dataset_properties=None): return {'shortname': 'SGD Regressor', 'name': 'Stochastic Gradient Descent Regressor', 'handles_regression': True, 'handles_classification': False, 'handles_multiclass': False, 'handles_multilabel': False, 'is_deterministic': True, 'handles_sparse': True, 'input': (DENSE, SPARSE, UNSIGNED_DATA), 'output': (PREDICTIONS,), } @staticmethod def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default_value="squared_loss") penalty = CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default_value="l2") alpha = UniformFloatHyperparameter( "alpha", 1e-7, 1e-1, log=True, default_value=0.0001) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 1e-9, 1., log=True, default_value=0.15) fit_intercept = UnParametrizedHyperparameter( "fit_intercept", "True") tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) epsilon = UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default_value=0.1, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default_value="invscaling") eta0 = UniformFloatHyperparameter( "eta0", 1e-7, 1e-1, default_value=0.01, log=True) power_t = UniformFloatHyperparameter( "power_t", 1e-5, 1, default_value=0.25) average = CategoricalHyperparameter( "average", ["False", "True"], default_value="False") cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate, eta0, power_t, average]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition(epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 is only relevant if learning_rate!='optimal' according to code # https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/ # linear_model/sgd_fast.pyx#L603 eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"]) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con]) return cs
from sklearn.linear_model.stochastic_gradient import SGDRegressor x_train = [[1, 0., 3], [1, 1., 3], [1, 2., 3], [1, 3., 2], [1, 4., 4]] y_train = [95.364, 97.217205, 75.195834, 60.105519, 49.342380] model = SGDRegressor(max_iter=5000000, alpha=0.00001) #[ 45.71878249 -13.02758034 1.14608487] model.fit(x_train, y_train) print(model.coef_) print(model.intercept_)
#%%观察数据 # 定义绘图辅助函数 def plt_helper(label, title, xlabel='x 轴', ylabel='y 轴'): fig = plt.figure() ax = fig.add_subplot(111, label=label) ax.set_title(title, fontproperties=myfont) ax.set_xlabel(xlabel, fontproperties=myfont) ax.set_ylabel(ylabel, fontproperties=myfont) ax.grid(True) return ax ax1 = plt_helper('ax1', '观察模拟数据的分布') ax1.plot(X[:, 0], y, 'r*') #%% linear_SGD = SGDRegressor(loss='squared_loss', max_iter=100) linear_SGD.fit(train_x, train_y) y_SGD = linear_SGD.predict(test_x) linear_rg = LinearRegression( fit_intercept=True, #计算截距 normalize=False, #回归之前不对数据集进行规范化处理 copy_X=True, #复制X,不会对X的原始值产生影响 n_jobs=-1) #使用所有的CPU linear_rg.fit(train_x, train_y) y_rg = linear_rg.predict(test_x) print('模拟数据参数', coef) print('SGDRegressor模型参数', linear_SGD.coef_) print('LinearRegression模型参数', linear_rg.coef_)
build_housing( AdaBoostRegressor(DecisionTreeRegressor(random_state=13, min_samples_leaf=5), random_state=13, n_estimators=17), "AdaBoostHousing") build_housing(KNeighborsRegressor(), "KNNHousing", with_kneighbors=True) build_housing( MLPRegressor(activation="tanh", hidden_layer_sizes=(26, ), solver="lbfgs", random_state=13, tol=0.001, max_iter=1000), "MLPHousing") build_housing(SGDRegressor(random_state=13), "SGDHousing") build_housing(SVR(), "SVRHousing") build_housing(LinearSVR(random_state=13), "LinearSVRHousing") build_housing(NuSVR(), "NuSVRHousing") # # Anomaly detection # def build_iforest_housing_anomaly(iforest, name): mapper = DataFrameMapper([(housing_X.columns.values, ContinuousDomain())]) pipeline = PMMLPipeline([("mapper", mapper), ("estimator", iforest)]) pipeline.fit(housing_X) store_pkl(pipeline, name + ".pkl") decisionFunction = DataFrame(pipeline.decision_function(housing_X),
class SGD(AutoSklearnRegressionAlgorithm): def __init__(self, loss, penalty, alpha, fit_intercept, n_iter, learning_rate, l1_ratio=0.15, epsilon=0.1, eta0=0.01, power_t=0.5, average=False, random_state=None): self.loss = loss self.penalty = penalty self.alpha = alpha self.fit_intercept = fit_intercept self.n_iter = n_iter self.learning_rate = learning_rate self.l1_ratio = l1_ratio self.epsilon = epsilon self.eta0 = eta0 self.power_t = power_t self.random_state = random_state self.average = average self.estimator = None self.scaler = None def fit(self, X, y): self.iterative_fit(X, y, n_iter=1, refit=True) while not self.configuration_fully_fitted(): self.iterative_fit(X, y, n_iter=1) return self def iterative_fit(self, X, y, n_iter=1, refit=False): from sklearn.linear_model.stochastic_gradient import SGDRegressor import sklearn.preprocessing if refit: self.estimator = None self.scaler = None if self.estimator is None: self.alpha = float(self.alpha) self.fit_intercept = self.fit_intercept == 'True' self.n_iter = int(self.n_iter) self.l1_ratio = float( self.l1_ratio) if self.l1_ratio is not None else 0.15 self.epsilon = float( self.epsilon) if self.epsilon is not None else 0.1 self.eta0 = float(self.eta0) self.power_t = float( self.power_t) if self.power_t is not None else 0.25 self.average = self.average == 'True' self.estimator = SGDRegressor(loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, n_iter=n_iter, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=self.epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state) self.scaler = sklearn.preprocessing.StandardScaler(copy=True) self.scaler.fit(y.reshape((-1, 1))) else: self.estimator.n_iter += n_iter Y_scaled = self.scaler.transform(y.reshape((-1, 1))).ravel() self.estimator.partial_fit(X, Y_scaled) if self.estimator.n_iter >= self.n_iter: self.fully_fit_ = True return self def configuration_fully_fitted(self): if self.estimator is None: return False elif not hasattr(self, 'fully_fit_'): return False else: return self.fully_fit_ def predict(self, X): if self.estimator is None: raise NotImplementedError() Y_pred = self.estimator.predict(X) return self.scaler.inverse_transform(Y_pred) @staticmethod def get_properties(dataset_properties=None): return {'shortname': 'SGD Regressor', 'name': 'Stochastic Gradient Descent Regressor', 'handles_missing_values': False, 'handles_nominal_values': False, 'handles_numerical_features': True, 'prefers_data_scaled': True, 'prefers_data_normalized': True, 'handles_regression': True, 'handles_classification': False, 'handles_multiclass': False, 'handles_multilabel': False, 'is_deterministic': True, 'handles_sparse': True, 'input': (DENSE, SPARSE, UNSIGNED_DATA), 'output': (PREDICTIONS,), # TODO find out what is best used here! 'preferred_dtype': None} @staticmethod def get_hyperparameter_search_space(dataset_properties=None): cs = ConfigurationSpace() loss = CategoricalHyperparameter("loss", ["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"], default="squared_loss") penalty = CategoricalHyperparameter( "penalty", ["l1", "l2", "elasticnet"], default="l2") alpha = UniformFloatHyperparameter( "alpha", 10e-7, 1e-1, log=True, default=0.01) l1_ratio = UniformFloatHyperparameter( "l1_ratio", 1e-9, 1., log=True, default=0.15) fit_intercept = UnParametrizedHyperparameter( "fit_intercept", "True") n_iter = UniformIntegerHyperparameter( "n_iter", 5, 1000, log=True, default=20) epsilon = UniformFloatHyperparameter( "epsilon", 1e-5, 1e-1, default=1e-4, log=True) learning_rate = CategoricalHyperparameter( "learning_rate", ["optimal", "invscaling", "constant"], default="optimal") eta0 = UniformFloatHyperparameter( "eta0", 10 ** -7, 0.1, default=0.01) power_t = UniformFloatHyperparameter( "power_t", 1e-5, 1, default=0.5) average = CategoricalHyperparameter( "average", ["False", "True"], default="False") cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept, n_iter, epsilon, learning_rate, eta0, power_t, average]) # TODO add passive/aggressive here, although not properly documented? elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet") epsilon_condition = InCondition(epsilon, loss, ["huber", "epsilon_insensitive", "squared_epsilon_insensitive"]) # eta0 seems to be always active according to the source code; when # learning_rate is set to optimial, eta0 is the starting value: # https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx # eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling") #eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant") #eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant) power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling") cs.add_conditions([elasticnet, epsilon_condition, power_t_condition]) return cs
def iterative_fit(self, X, y, n_iter=2, refit=False): from sklearn.linear_model.stochastic_gradient import SGDRegressor import sklearn.preprocessing # Need to fit at least two iterations, otherwise early stopping will not # work because we cannot determine whether the algorithm actually # converged. The only way of finding this out is if the sgd spends less # iterations than max_iter. If max_iter == 1, it has to spend at least # one iteration and will always spend at least one iteration, so we # cannot know about convergence. n_iter = max(n_iter, 2) if refit: self.estimator = None self.scaler = None if self.estimator is None: self.alpha = float(self.alpha) self.fit_intercept = check_for_bool(self.fit_intercept) self.tol = float(self.tol) self.l1_ratio = float( self.l1_ratio) if self.l1_ratio is not None else 0.15 self.epsilon = float( self.epsilon) if self.epsilon is not None else 0.1 self.eta0 = float(self.eta0) self.power_t = float( self.power_t) if self.power_t is not None else 0.25 self.average = check_for_bool(self.average) self.estimator = SGDRegressor(loss=self.loss, penalty=self.penalty, alpha=self.alpha, fit_intercept=self.fit_intercept, max_iter=n_iter, tol=self.tol, learning_rate=self.learning_rate, l1_ratio=self.l1_ratio, epsilon=self.epsilon, eta0=self.eta0, power_t=self.power_t, shuffle=True, average=self.average, random_state=self.random_state, warm_start=True) self.scaler = sklearn.preprocessing.StandardScaler(copy=True) self.scaler.fit(y.reshape((-1, 1))) Y_scaled = self.scaler.transform(y.reshape((-1, 1))).ravel() self.estimator.fit(X, Y_scaled) else: self.estimator.max_iter += n_iter self.estimator.max_iter = min(self.estimator.max_iter, 1000) Y_scaled = self.scaler.transform(y.reshape((-1, 1))).ravel() self.estimator._validate_params() self.estimator._partial_fit( X, Y_scaled, alpha=self.estimator.alpha, C=1.0, loss=self.estimator.loss, learning_rate=self.estimator.learning_rate, max_iter=n_iter, sample_weight=None, coef_init=None, intercept_init=None) if self.estimator._max_iter >= 1000 or n_iter > self.estimator.n_iter_: self.fully_fit_ = True return self
np.random.shuffle(inds) coef[inds[n_features/2:]] = 0 # sparsify coef print("true coef sparsity: %f" % sparsity_ratio(coef)) y = np.dot(X, coef) # add noise y += 0.01 * np.random.normal((n_samples,)) # Split data in train set and test set n_samples = X.shape[0] X_train, y_train = X[:n_samples / 2], y[:n_samples / 2] X_test, y_test = X[n_samples / 2:], y[n_samples / 2:] print("test data sparsity: %f" % sparsity_ratio(X_test)) ############################################################################### clf = SGDRegressor(penalty='l1', alpha=.2, fit_intercept=True, n_iter=2000) clf.fit(X_train, y_train) print("model sparsity: %f" % sparsity_ratio(clf.coef_)) def benchmark_dense_predict(): for _ in range(300): clf.predict(X_test) def benchmark_sparse_predict(): X_test_sparse = csr_matrix(X_test) for _ in range(300): clf.predict(X_test_sparse)
]) + 1 #check how far is that index in the dropdown list and return that value def average_lowest_correct(list_of_trues, list_of_preds): length = len(list_of_trues) # number of data points return np.mean([ lowest_correct(list(list_of_trues.iloc[i]), list(list_of_preds[i])) for i in range(length) ]) # Top four models selected formatted as a pipteline to be used for gridsearch model_1 = Pipeline([('md1', MultiOutputRegressor(Ridge()))]) model_2 = Pipeline([('md2', MultiOutputRegressor(KernelRidge()))]) model_3 = Pipeline([('md3', MultiOutputRegressor(LinearSVR()))]) model_4 = Pipeline([('md4', MultiOutputRegressor(SGDRegressor()))]) # Dictionary of all the variable hyperparameters for all four models. Except of the SGD regressor, the hyperparameter list is complete. model_params = { 'Multi_Ridge': { 'model': model_1, 'params': { 'md1__estimator__normalize': [True, False], 'md1__estimator__fit_intercept': [True, False], 'md1__estimator__solver': ['svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'], 'md1__estimator__alpha': [i for i in range(10, 110, 10)], 'md1__estimator__max_iter': [1000, 2000, 3000] } }, 'Multi_KernelRidge': {
] classifiers = [ RandomForestRegressor(n_estimators=200, n_jobs=5, random_state=randomstate), ExtraTreesRegressor(n_estimators=200, n_jobs=5, random_state=randomstate), # GradientBoostingRegressor(random_state=randomstate), # learning_rate is a hyper-parameter in the range (0.0, 1.0] # HistGradientBoostingClassifier(random_state=randomstate), # learning_rate is a hyper-parameter in the range (0.0, 1.0] AdaBoostRegressor(n_estimators=200, random_state=randomstate), GaussianProcessRegressor(normalize_y=True), ARDRegression(), # HuberRegressor(), # epsilon: greater than 1.0, default 1.35 LinearRegression(n_jobs=5), PassiveAggressiveRegressor( random_state=randomstate), # C: 0.25, 0.5, 1, 5, 10 SGDRegressor(random_state=randomstate), TheilSenRegressor(n_jobs=5, random_state=randomstate), RANSACRegressor(random_state=randomstate), KNeighborsRegressor( weights='distance'), # n_neighbors: 3, 6, 9, 12, 15, 20 RadiusNeighborsRegressor(weights='distance'), # radius: 1, 2, 5, 10, 15 MLPRegressor(max_iter=10000000, random_state=randomstate), DecisionTreeRegressor( random_state=randomstate), # max_depth = 2, 3, 4, 6, 8 ExtraTreeRegressor(random_state=randomstate), # max_depth = 2, 3, 4, 6, 8 SVR() # C: 0.25, 0.5, 1, 5, 10 ] selectors = [ reliefF.reliefF, fisher_score.fisher_score,
y = scaler.fit_transform(Y) ''' regularized training error given by: E(w, b) = 1/n * sum(L(yi, f(xi))) + alpha * R(w) Note: L is loss function, R(w) is regularization term (penalty) For Elastic Net R(w): R(w) = p/2 * sum(wi^2) + (1 - p) * |wi| where p is given by 1 - l1_ratio For inverse scaling learning_rate: lr = eta0 / t^power_t ''' regr = SGDRegressor(penalty = 'elasticnet', alpha = 0.0001, l1_ratio = 0.25, learning_rate = 'invscaling', eta0 = 0.01, power_t = 0.25, loss = 'epsilon_insensitive', epsilon = 0.1, shuffle = True, fit_intercept = True, n_iter = 1000000, average = False, verbose = 0) regr.fit(x, y) data_pred = regr.predict(x) y_pred = scaler.inverse_transform(data_pred) print('coefficients: \n', regr.coef_) #if data is expected to be already centered then intercept_ is not needed print('intercept: \n', regr.intercept_) #Calculate mean squared error print('Mean Squared Error: %.4f' % mean_squared_error(y, data_pred))
# Apply scaler on training and test data standardized_X_train = X_scaler.transform(X_train) standardized_y_train = y_scaler.transform(y_train.values.reshape(-1, 1)).ravel() standardized_X_test = X_scaler.transform(X_test) standardized_y_test = y_scaler.transform(y_test.values.reshape(-1, 1)).ravel() # Check print("mean:", np.mean(standardized_X_train, axis=0), np.mean(standardized_y_train, axis=0)) # mean should be ~0 print("std:", np.std(standardized_X_train, axis=0), np.std(standardized_y_train, axis=0)) # std should be 1 # Initialize the model lm = SGDRegressor(loss="squared_loss", penalty="none", max_iter=args.num_epochs) # Train lm.fit(X=standardized_X_train, y=standardized_y_train) # Predictions (unstandardize them) pred_train = (lm.predict(standardized_X_train) * np.sqrt(y_scaler.var_)) + y_scaler.mean_ pred_test = (lm.predict(standardized_X_test) * np.sqrt(y_scaler.var_)) + y_scaler.mean_ # Train and test MSE train_mse = np.mean((y_train - pred_train)**2) test_mse = np.mean((y_test - pred_test)**2) print("train_MSE: {0:.2f}, test_MSE: {1:.2f}".format(train_mse, test_mse))
# ############################################################################# # Benchmark bulk/atomic prediction speed for various regressors configuration = { 'n_train': int(1e3), 'n_test': int(1e2), 'n_features': int(1e2), 'estimators': [ { 'name': 'Linear Model', 'instance': SGDRegressor(penalty='elasticnet', alpha=0.01, l1_ratio=0.25, tol=1e-4), 'complexity_label': 'non-zero coefficients', 'complexity_computer': lambda clf: np.count_nonzero(clf.coef_) }, { 'name': 'RandomForest', 'instance': RandomForestRegressor(), 'complexity_label': 'estimators', 'complexity_computer': lambda clf: clf.n_estimators }, { 'name': 'SVR', 'instance': SVR(kernel='rbf'),