def smac_train_test(trainable, X_train, y_train): try: cv_score, logloss, execution_time = cross_val_score_track_trials( trainable, X_train, y_train, cv=self.cv, scoring=self.scoring) logger.debug("Successful trial of SMAC") except BaseException as e: #If there is any error in cross validation, use the score based on a random train-test split as the evaluation criterion if self.handle_cv_failure: X_train_part, X_validation, y_train_part, y_validation = train_test_split( X_train, y_train, test_size=0.20) start = time.time() trained = trainable.fit(X_train_part, y_train_part) scorer = check_scoring(trainable, scoring=self.scoring) cv_score = scorer(trained, X_validation, y_validation) execution_time = time.time() - start y_pred_proba = trained.predict_proba(X_validation) try: logloss = log_loss(y_true=y_validation, y_pred=y_pred_proba) except BaseException: logloss = 0 logger.debug("Warning, log loss cannot be computed") else: logger.debug("Error {} with pipeline:{}".format( e, trainable.to_json())) raise e return cv_score, logloss, execution_time
def hyperopt_train_test(params, X_train, y_train): warnings.filterwarnings("ignore") reg = create_instance_from_hyperopt_search_space( self.estimator, params) try: cv_score, _, execution_time = cross_val_score_track_trials( reg, X_train, y_train, cv=KFold(self.cv), scoring=self.scoring) logger.debug("Successful trial of hyperopt") except BaseException as e: #If there is any error in cross validation, use the accuracy based on a random train-test split as the evaluation criterion if self.handle_cv_failure: X_train_part, X_validation, y_train_part, y_validation = train_test_split( X_train, y_train, test_size=0.20) start = time.time() reg_trained = reg.fit(X_train_part, y_train_part) scorer = check_scoring(reg, scoring=self.scoring) cv_score = scorer(reg_trained, X_validation, y_validation) execution_time = time.time() - start else: logger.debug(e) logger.debug("Error {} with pipeline:{}".format( e, reg.to_json())) raise e return cv_score, execution_time
def hyperopt_train_test(params, X_train, y_train): warnings.filterwarnings("ignore") clf = create_instance_from_hyperopt_search_space( self.model, params) try: cv_score, logloss, execution_time = cross_val_score_track_trials( clf, X_train, y_train, cv=self.cv) logger.debug("Successful trial of hyperopt") except BaseException as e: #If there is any error in cross validation, use the accuracy based on a random train-test split as the evaluation criterion if self.handle_cv_failure: X_train_part, X_validation, y_train_part, y_validation = train_test_split( X_train, y_train, test_size=0.20) start = time.time() clf_trained = clf.fit(X_train_part, y_train_part) predictions = clf_trained.predict(X_validation) execution_time = time.time() - start y_pred_proba = clf_trained.predict_proba(X_validation) try: logloss = log_loss(y_true=y_validation, y_pred=y_pred_proba) except BaseException: logloss = 0 logger.debug("Warning, log loss cannot be computed") cv_score = accuracy_score( y_validation, [round(pred) for pred in predictions]) else: logger.debug(e) logger.debug("Error {} with pipeline:{}".format( e, clf.to_json())) raise e #print("TRIALS") #print(json.dumps(self.get_trials().trials, default = myconverter, indent=4)) return cv_score, logloss, execution_time
def hyperopt_train_test(params, X_train, y_train): warnings.filterwarnings("ignore") clf = create_instance_from_hyperopt_search_space( self.estimator, params) try: cv_score, logloss, execution_time = cross_val_score_track_trials( clf, X_train, y_train, cv=self.cv, scoring=self.scoring) logger.debug("Successful trial of hyperopt") except BaseException as e: #If there is any error in cross validation, use the score based on a random train-test split as the evaluation criterion if self.handle_cv_failure: X_train_part, X_validation, y_train_part, y_validation = train_test_split( X_train, y_train, test_size=0.20) start = time.time() clf_trained = clf.fit(X_train_part, y_train_part) #predictions = clf_trained.predict(X_validation) scorer = check_scoring(clf, scoring=self.scoring) cv_score = scorer(clf_trained, X_validation, y_validation) execution_time = time.time() - start y_pred_proba = clf_trained.predict_proba(X_validation) try: logloss = log_loss(y_true=y_validation, y_pred=y_pred_proba) except BaseException: logloss = 0 logger.debug("Warning, log loss cannot be computed") else: logger.debug(e) logger.debug("Error {} with pipeline:{}".format( e, clf.to_json())) raise e return cv_score, logloss, execution_time
def hyperopt_train_test(params, X_train, y_train): warnings.filterwarnings("ignore") trainable = create_instance_from_hyperopt_search_space( self.estimator, params ) try: cv_score, logloss, execution_time = cross_val_score_track_trials( trainable, X_train, y_train, cv=self.cv, scoring=self.scoring, args_to_scorer=self.args_to_scorer, ) logger.debug( "Successful trial of hyperopt with hyperparameters:{}".format( params ) ) except BaseException as e: # If there is any error in cross validation, use the score based on a random train-test split as the evaluation criterion if self.handle_cv_failure and trainable is not None: ( X_train_part, X_validation, y_train_part, y_validation, ) = train_test_split(X_train, y_train, test_size=0.20) start = time.time() trained = trainable.fit(X_train_part, y_train_part, **fit_params) scorer = check_scoring(trainable, scoring=self.scoring) cv_score = scorer( trained, X_validation, y_validation, **self.args_to_scorer ) execution_time = time.time() - start y_pred_proba = trained.predict_proba(X_validation) try: logloss = log_loss(y_true=y_validation, y_pred=y_pred_proba) except BaseException: logloss = 0 logger.debug("Warning, log loss cannot be computed") else: logger.debug(e) if trainable is None: logger.debug( "Error {} with uncreatable pipeline with parameters:{}".format( e, lale.pretty_print.hyperparams_to_string(params) ) ) else: logger.debug( "Error {} with pipeline:{}".format(e, trainable.to_json()) ) raise e return cv_score, logloss, execution_time