def train_gama(X_train, X_test, y_train, y_test, mtype, common_name_model, problemtype, classes, default_featurenames, transform_model, settings, model_session): model_name = common_name_model + '.pickle' files = list() if mtype in ['c']: automl = GamaClassifier(max_total_time=180, keep_analysis_log=None) print( "Starting GAMA `fit` - usually takes around 3 minutes but can take longer for large datasets" ) automl.fit(X_train, y_train) label_predictions = automl.predict(X_test) probability_predictions = automl.predict_proba(X_test) accuracy = accuracy_score(y_test, label_predictions) log_loss_pred = log_loss(y_test, probability_predictions) log_loss_score = automl.score(X_test, y_test) print('accuracy:', accuracy) print('log loss pred:', log_loss_pred) print('log_loss_score', log_loss_score) elif mtype in ['regression', 'r']: automl = GamaRegressor(max_total_time=180, keep_analysis_log=None, n_jobs=1) print( "Starting GAMA `fit` - usually takes around 3 minutes but can take longer for large datasets" ) automl.fit(X_train, y_train) predictions = automl.predict(X_test) mse_error = mean_squared_error(y_test, predictions) print("MSE:", mse_error) # SAVE ML MODEL modelfile = open(model_name, 'wb') pickle.dump(automl, modelfile) modelfile.close() files.append(model_name) model_dir = os.getcwd() return model_name, model_dir, files
from sklearn.datasets import load_breast_cancer from sklearn.model_selection import train_test_split from sklearn.metrics import log_loss, accuracy_score from gama import GamaClassifier if __name__ == '__main__': X, y = load_breast_cancer(return_X_y=True) X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) automl = GamaClassifier(max_total_time=180, keep_analysis_log=None, n_jobs=1) print("Starting `fit` which will take roughly 3 minutes.") automl.fit(X_train, y_train) label_predictions = automl.predict(X_test) probability_predictions = automl.predict_proba(X_test) print('accuracy:', accuracy_score(y_test, label_predictions)) print('log loss:', log_loss(y_test, probability_predictions))
def _test_dataset_problem( data, metric: str, arff: bool = False, y_type: Type = pd.DataFrame, search: BaseSearch = AsyncEA(), missing_values: bool = False, max_time: int = 60, ): """ :param data: :param metric: :param arff: :param y_type: pd.DataFrame, pd.Series, np.ndarray or str :return: """ gama = GamaClassifier( random_state=0, max_total_time=max_time, scoring=metric, search=search, n_jobs=1, post_processing=EnsemblePostProcessing(ensemble_size=5), store="nothing", ) if arff: train_path = f"tests/data/{data['name']}_train.arff" test_path = f"tests/data/{data['name']}_test.arff" X, y = data["load"](return_X_y=True) X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) y_test = [str(val) for val in y_test] with Stopwatch() as sw: gama.fit_from_file(train_path, target_column=data["target"]) class_predictions = gama.predict_from_file( test_path, target_column=data["target"]) class_probabilities = gama.predict_proba_from_file( test_path, target_column=data["target"]) gama_score = gama.score_from_file(test_path) else: X, y = data["load"](return_X_y=True) if y_type == str: databunch = data["load"]() y = np.asarray( [databunch.target_names[c_i] for c_i in databunch.target]) if y_type in [pd.Series, pd.DataFrame]: y = y_type(y) X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) if missing_values: X_train[1:300:2, 0] = X_train[2:300:5, 1] = float("NaN") X_test[1:100:2, 0] = X_test[2:100:5, 1] = float("NaN") with Stopwatch() as sw: gama.fit(X_train, y_train) class_predictions = gama.predict(X_test) class_probabilities = gama.predict_proba(X_test) gama_score = gama.score(X_test, y_test) assert (60 * FIT_TIME_MARGIN > sw.elapsed_time), "fit must stay within 110% of allotted time." assert isinstance(class_predictions, np.ndarray), "predictions should be numpy arrays." assert ( data["test_size"], ) == class_predictions.shape, "predict should return (N,) shaped array." accuracy = accuracy_score(y_test, class_predictions) # Majority classifier on this split achieves 0.6293706293706294 print(data["name"], metric, "accuracy:", accuracy) assert (data["base_accuracy"] <= accuracy ), "predictions should be at least as good as majority class." assert isinstance( class_probabilities, np.ndarray), "probability predictions should be numpy arrays." assert (data["test_size"], data["n_classes"]) == class_probabilities.shape, ( "predict_proba should return" " (N,K) shaped array.") # Majority classifier on this split achieves 12.80138131184662 logloss = log_loss(y_test, class_probabilities) print(data["name"], metric, "log-loss:", logloss) assert (data["base_log_loss"] >= logloss ), "predictions should be at least as good as majority class." score_to_match = logloss if metric == "neg_log_loss" else accuracy assert score_to_match == pytest.approx(gama_score) gama.cleanup("all") return gama
# In[6]: #Initialization cls = GamaClassifier(max_total_time=3600, keep_analysis_log=None, n_jobs=1, scoring='accuracy', post_processing_method=EnsemblePostProcessing()) X = B[0].iloc[:, 0:-1] y = B[0].iloc[:, -1] print("Starting `fit`") cls.fit(X, y) anytime_model = cls #Prequential evaluation for i in range(1, n): #Test on next batch for accuracy X = B[i].iloc[:, 0:-1] y = B[i].iloc[:, -1] y_hat = cls.predict(X) accuracy = sklearn.metrics.accuracy_score(y, y_hat) print("Test batch %d - Test score %f\n" % (i, accuracy)) # In[ ]:
X_train = B[start - 1].iloc[:, 0:-1] y_train = B[start - 1].iloc[:, -1] print("Starting to `fit`") cls.fit(X_train, y_train, warm_start=True) anytime_model = cls #Prequential evaluation for i in range(start, n): #Test on next batch for accuracy X_test = B[i].iloc[:, 0:-1] y_test = B[i].iloc[:, -1] y_hat = cls.predict(X_test) b_acc = sklearn.metrics.balanced_accuracy_score( y_test, y_hat) #equivalent to ROC_AUC in binary case acc = sklearn.metrics.accuracy_score(y_test, y_hat) print("Test batch %d - Balanced accuracy %f - Accuracy %f\n" % (i, b_acc, acc)) #Check for drift drift_in_batch = 0 for j in range(0, len(B[i])): drift_detector.add_element(int(not y_test.iloc[j] == y_hat[j])) if drift_detector.detected_change(): print('Drift in performance detected at sample {}'.format(j)) drift_in_batch = 1
def _test_dataset_problem(data, metric: str, arff: bool = False, y_type: Type = pd.DataFrame, search: BaseSearch = AsyncEA(), missing_values: bool = False, max_time: int = 60): """ :param data: :param metric: :param arff: :param y_type: pd.DataFrame, pd.Series, np.ndarray or str :return: """ gama = GamaClassifier( random_state=0, max_total_time=max_time, scoring=metric, search_method=search, n_jobs=1, post_processing_method=EnsemblePostProcessing(ensemble_size=5)) if arff: train_path = 'tests/data/{}_train.arff'.format(data['name']) test_path = 'tests/data/{}_test.arff'.format(data['name']) X, y = data['load'](return_X_y=True) X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) y_test = [str(val) for val in y_test] with Stopwatch() as sw: gama.fit_arff(train_path, target_column=data['target']) class_predictions = gama.predict_arff(test_path, target_column=data['target']) class_probabilities = gama.predict_proba_arff( test_path, target_column=data['target']) gama_score = gama.score_arff(test_path) else: X, y = data['load'](return_X_y=True) if y_type == str: databunch = data['load']() y = np.asarray( [databunch.target_names[c_i] for c_i in databunch.target]) if y_type in [pd.Series, pd.DataFrame]: y = y_type(y) X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=0) if missing_values: X_train[1:300:2, 0] = X_train[2:300:5, 1] = float("NaN") X_test[1:100:2, 0] = X_test[2:100:5, 1] = float("NaN") with Stopwatch() as sw: gama.fit(X_train, y_train) class_predictions = gama.predict(X_test) class_probabilities = gama.predict_proba(X_test) gama_score = gama.score(X_test, y_test) assert 60 * FIT_TIME_MARGIN > sw.elapsed_time, 'fit must stay within 110% of allotted time.' assert isinstance(class_predictions, np.ndarray), 'predictions should be numpy arrays.' assert ( data['test_size'], ) == class_predictions.shape, 'predict should return (N,) shaped array.' accuracy = accuracy_score(y_test, class_predictions) # Majority classifier on this split achieves 0.6293706293706294 print(data['name'], metric, 'accuracy:', accuracy) assert data[ 'base_accuracy'] <= accuracy, 'predictions should be at least as good as majority class.' assert isinstance( class_probabilities, np.ndarray), 'probability predictions should be numpy arrays.' assert (data['test_size'], data['n_classes']) == class_probabilities.shape, ( 'predict_proba should return' ' (N,K) shaped array.') # Majority classifier on this split achieves 12.80138131184662 logloss = log_loss(y_test, class_probabilities) print(data['name'], metric, 'log-loss:', logloss) assert data[ 'base_log_loss'] >= logloss, 'predictions should be at least as good as majority class.' score_to_match = logloss if metric == 'log_loss' else accuracy assert score_to_match == pytest.approx(gama_score)