def objective(trial): save_steps = 50 estimator.TrainParams['learning_rate'] = trial.suggest_loguniform( "learning_rate", 1e-5, 1e-2) clf = estimator.get_model(model_dir) train_spec, eval_spec, train_input_fn, eval_input_fn = \ estimator.get_train_and_eval_specs_inputs( train_x, train_y, test_x, test_y, epochs, batch_size) # Create hooks early_stopping_hook = tf.estimator.experimental.stop_if_no_decrease_hook( clf, "loss", save_steps) optuna_pruning_hook = TensorFlowPruningHook( trial=trial, estimator=clf, metric="loss", run_every_steps=10, ) hooks = [] #hooks = [optuna_pruning_hook] hooks = [early_stopping_hook, optuna_pruning_hook] # Run training and evaluation tf.estimator.train_and_evaluate(clf, train_spec, eval_spec) result = clf.evaluate(input_fn=eval_input_fn) #result = clf.evaluate(input_fn=eval_input_fn, steps=100) print(result) return result['loss']
def xgboost_trial(trial: trial) -> dict: """ A xgboost trial function te be run inside optuna optimization function :param trial: trial object from optuna :return: Returns dictionary with trial params """ params = { "subsample": trial.suggest_discrete_uniform("subsample", 0.1, 1, 0.1), "colsample_bytree": trial.suggest_discrete_uniform("colsample_bytree", 0.6, 1, 0.1), "colsample_bylevel": trial.suggest_discrete_uniform("colsample_bylevel", 0.6, 1, 0.1), "min_child_weight": trial.suggest_int("min_child_weight", 1, 20), "max_depth": trial.suggest_int("max_depth", 3, 12), "max_delta_step": trial.suggest_discrete_uniform("max_delta_step", 0.1, 10, 0.1), "learning_rate": trial.suggest_loguniform("learning_rate", 0.01, 0.1), "n_estimators": trial.suggest_int("n_estimators", 100, 500), "gamma": trial.suggest_discrete_uniform("gamma", 0.1, 30, 0.1) } return params
def f(trial): # type: (optuna.trial.Trial) -> float x = trial.suggest_uniform('x', -10, 10) y = trial.suggest_loguniform('y', 10, 20) z = trial.suggest_categorical('z', (10, 20.5, 30)) return x**2 + y**2 + z
def f(trial: optuna.trial.Trial) -> float: x = trial.suggest_uniform("x", -10, 10) y = trial.suggest_loguniform("y", 10, 20) z = trial.suggest_categorical("z", (10.0, 20.5, 30.0)) assert isinstance(z, float) return x**2 + y**2 + z
def objective(trial: BaseTrial) -> float: a = trial.suggest_uniform("a", 0.0, 10.0) b = trial.suggest_loguniform("b", 0.1, 10.0) c = trial.suggest_discrete_uniform("c", 0.0, 10.0, 1.0) d = trial.suggest_int("d", 0, 10) e = trial.suggest_categorical("e", [0, 1, 2]) f = trial.suggest_int("f", 1, 10, log=True) assert isinstance(e, int) return a + b + c + d + e + f
def __call__(self, trial): # optional variables sampling ( to implement in the future) # if x_var_num_range is not None: # number_of_var = list(range(self.x_var_num_range[0], self.x_var_num_range[1]+1)) # x_var = np.random.choice(list(x_var.columns), number_of_var) # else: # x_var = x_train.columns if self.params is not None: params = exec(self.params) else: params = None if self.method == 'SVC': # support vectors machine SVM_params = { 'svc_c': trial.suggest_loguniform('svc_c', 1e-10, 1e10) } if params is not None: SVM_params.update(params) model = sklearn.svm.SVC(**SVM_params) elif self.method == 'RF': # random forest # https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier.feature_importances_ RF_params = { 'n_estimators': int(trial.suggest_uniform('n_estimators', 20, 300)), 'max_depth': int(trial.suggest_uniform('max_depth', 2, 50)), 'criterion': trial.suggest_categorical('criterion', ['gini', 'entropy']), 'bootstrap': trial.suggest_categorical('bootstrap', [True, False]), 'class_weight': trial.suggest_categorical( 'class_weight', ['balanced', 'balanced_subsample', None]) } if params is not None: RF_params.update(params) model = RandomForestClassifier(**RF_params, n_jobs=15) elif self.method == 'AB': # ada boost AB_params = { 'n_estimators': int(trial.suggest_uniform('n_estimators', 20, 300)), 'learning_rate': trial.suggest_uniform('learning_rate', 0.05, 0.3), 'algorithm': trial.suggest_categorical('algorithm', ['SAMME.R', 'SAMME']) } if params is not None: AB_params.update(params) model = AdaBoostClassifier(**AB_params) elif self.method == 'GB': # gradient boost GB_params = { 'n_estimators': int(trial.suggest_loguniform('n_estimators', 20, 300)), 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5), 'max_depth': int(trial.suggest_loguniform('max_depth', 2, 50)), 'loss': trial.suggest_categorical('loss', ['deviance', 'exponential']) } if params is not None: GB_params.update(params) model = GradientBoostingClassifier(**GB_params) elif self.method == 'NB': # naive bayes NB_params = { 'var_smoothing': trial.suggest_loguniform('var_smoothing', 1e-10, 1e-05) } if params is not None: NB_params.update(params) model = skl.naive_bayes.GaussianNB(**NB_params) elif self.method == 'KNN': # k-nearest neighbours KNN_params = { 'n_neighbors': int(trial.suggest_loguniform('n_neighbors', 3, 7)) } if params is not None: KNN_params.update(params) model = skl.neighbors.KNeighborsClassifier(**KNN_params) elif self.method == 'LR': # logistic retression LR_params = { 'penalty': trial.suggest_categorical('penalty', ['l1', 'l2']), 'fit_intercept': trial.suggest_categorical('fit_intercept', [True, False]) } if params is not None: LR_params.update(params) model = skl.linear_model.LogisticRegression(**LR_params) elif self.method == 'LGBM': # logistic retression LGBM_params = { 'penalty': trial.suggest_categorical('penalty', ['l1', 'l2']), 'fit_intercept': trial.suggest_categorical('fit_intercept', [True, False]) } if params is not None: LGBM_params.update(params) model = LGBMClassifier(**LGBM_params) elif self.method == 'XGB': # XGBoost # https://xgboost.readthedocs.io/en/latest/parameter.html XGB_params = { 'n_estimators': int(trial.suggest_loguniform('n_estimators', 5, 300)), 'booster': trial.suggest_categorical('booster', ['dart', 'gbtree']), 'eta': trial.suggest_loguniform('eta', 0.01, 0.5), 'max_depth': int(trial.suggest_loguniform('max_depth', 3, 30)) # ,'reg_lambda': trial.suggest_loguniform( 'reg_lambda', 0, 1) # ,'reg_alpha' : trial.suggest_loguniform( 'reg_alpha', 0, 1) } if params is not None: XGB_params.update(params) model = XGBClassifier(**XGB_params, nthread=15) elif self.method == 'CAT': # catboost # https://catboost.ai/docs/concepts/python-reference_parameters-list.html#python-reference_parameters-list CAT_params = { 'iterations': int(trial.suggest_loguniform('iterations', 20, 300)), 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5), 'depth': int(trial.suggest_loguniform('depth', 2, 16)) } if params is not None: CAT_params.update(params) model = CatBoostClassifier(**CAT_params, early_stopping_rounds=75, logging_level='Silent') # calibration (optional) if self.calibration_method is not None: model = CalibratedClassifierCV(model, cv=3, method=self.calibration_method) # model fit model.fit(self.x_train, self.y_train) # y labels y_labels = list(self.y_train.drop_duplicates()) y_labels.sort() self.y_labels = y_labels # model prediction and classification prob_train = pd.DataFrame(model.predict_proba(self.x_train), columns=[str(x) for x in model.classes_]) prob_train = prob_train.reset_index(drop=True) prob_test = pd.DataFrame(model.predict_proba(self.x_test), columns=[str(x) for x in model.classes_]) prob_test = prob_test.reset_index(drop=True) if self.priori is not None: for p in range(prob_test.shape[1]): var = prob_test.columns[p] priori_p = self.priori[var] prob_test.iloc[:, p] = prob_test.iloc[:, p] * priori_p prob_train.iloc[:, p] = prob_train.iloc[:, p] * priori_p prob_train_sum = prob_train.apply(lambda x: 1 / sum(x), axis=1) prob_test_sum = prob_test.apply(lambda x: 1 / sum(x), axis=1) for p in self.y_labels: prob_train.loc[:, p] = prob_train.loc[:, p] * prob_train_sum prob_test.loc[:, p] = prob_test.loc[:, p] * prob_test_sum if prob_test.shape[1] == 2 and self.threshold is not None: var_1 = list(self.threshold.keys())[0] var_2 = list(self.threshold.keys())[1] threshold_var_1 = list(self.threshold.values())[0] threshold_var_2 = list(self.threshold.values())[1] classification_train = pd.DataFrame([ var_1 if x >= threshold_var_1 else var_2 for x in prob_train[str(var_1)] ]) classification_test = pd.DataFrame([ var_1 if x >= threshold_var_1 else var_2 for x in prob_test[str(var_1)] ]) else: classification_train = pd.DataFrame(prob_train.idxmax( axis=1)) # zapis jako DataFrame by dalej zrobić 'concat' classification_test = pd.DataFrame(prob_test.idxmax( axis=1)) # zapis jako DataFrame by dalej zrobić 'concat' # selection and determining score function to optimize balanced_accuracy = skm.balanced_accuracy_score( y_true=self.y_test, y_pred=classification_test) accuracy = skm.accuracy_score(y_true=self.y_test, y_pred=classification_test) # ta funkcja nie jest napisane pod katem modeli multilabel aps = average_precision_score(y_true=self.y_test, y_score=prob_test[str(self.pos_label)], pos_label=str(self.pos_label)) skm.recall_score(y_true=self.y_test.astype(int), y_pred=classification_test.astype(int), average='binary', pos_label=self.pos_label) # recall try: if len(self.y_labels) < 3: recall_train = skm.recall_score( y_true=self.y_train.astype(int), y_pred=classification_train.astype(int), average='binary', pos_label=self.pos_label) recall_test = skm.recall_score( y_true=self.y_test.astype(int), y_pred=classification_test.astype(int), average='binary', pos_label=self.pos_label) else: recall_train = skm.recall_score( y_true=self.y_train.astype(str), y_pred=classification_train.astype(str), average='weighted') recall_test = skm.recall_score( y_true=self.y_test.astype(str), y_pred=classification_test.astype(str), average='weighted') except: recall_train = np.nan recall_test = np.nan # precision try: if len(self.y_labels) < 3: precision_train = skm.precision_score( y_true=self.y_train.astype(int), y_pred=classification_train.astype(int), average='binary', pos_label=self.pos_label) precision_test = skm.precision_score( y_true=self.y_test.astype(int), y_pred=classification_test.astype(int), average='binary', pos_label=self.pos_label) else: precision_train = skm.precision_score( y_true=self.y_train.astype(str), y_pred=classification_train.astype(str), average='weighted') precision_test = skm.precision_score( y_true=self.y_test.astype(str), y_pred=classification_test.astype(str), average='weighted') except: precision_train = np.nan precision_test = np.nan # f1 try: if len(self.y_labels) < 3: f1_train = skm.f1_score(y_true=self.y_train, y_pred=classification_train, average='binary', pos_label=str(self.pos_label)) f1_test = skm.f1_score(y_true=self.y_test, y_pred=classification_test, average='binary', pos_label=str(self.pos_label)) else: f1_train = skm.f1_score(y_true=self.y_train, y_pred=classification_train, average='weighted') f1_test = skm.f1_score(y_true=self.y_test, y_pred=classification_test, average='weighted') except: f1_train = np.nan f1_test = np.nan scores = pd.DataFrame([[ balanced_accuracy, accuracy, recall_train, recall_test, precision_train, precision_test, f1_train, f1_test, aps ]], columns=[ 'balanced_accuracy', 'accuracy', 'recall_train', 'recall_test', 'precision_train', 'precision_test', 'f1_train', 'f1_test', 'aps' ]) self.scores = pd.concat([self.scores, scores]) # zeracanie scoru którego 'optuna' używa do optymalizacji grida (możemy chyba tylko jedną wartość zwrócić. if self.opt_function == 'balanced_accuracy': return (balanced_accuracy) elif self.opt_function == 'aps': return (aps) elif self.opt_function == 'accuracy': return (accuracy) elif self.opt_function == 'recall': return (recall_test) elif self.opt_function == 'precision': return (precision_test)