def train_gama(X_train, X_test, y_train, y_test, mtype, common_name_model,
problemtype, classes, default_featurenames, transform_model,
settings, model_session):
model_name = common_name_model + '.pickle'
files = list()
if mtype in ['c']:
automl = GamaClassifier(max_total_time=180, keep_analysis_log=None)
print(
"Starting GAMA `fit` - usually takes around 3 minutes but can take longer for large datasets"
)
automl.fit(X_train, y_train)
label_predictions = automl.predict(X_test)
probability_predictions = automl.predict_proba(X_test)
accuracy = accuracy_score(y_test, label_predictions)
log_loss_pred = log_loss(y_test, probability_predictions)
log_loss_score = automl.score(X_test, y_test)
print('accuracy:', accuracy)
print('log loss pred:', log_loss_pred)
print('log_loss_score', log_loss_score)
elif mtype in ['regression', 'r']:
automl = GamaRegressor(max_total_time=180,
keep_analysis_log=None,
n_jobs=1)
print(
"Starting GAMA `fit` - usually takes around 3 minutes but can take longer for large datasets"
)
automl.fit(X_train, y_train)
predictions = automl.predict(X_test)
mse_error = mean_squared_error(y_test, predictions)
print("MSE:", mse_error)
# SAVE ML MODEL
modelfile = open(model_name, 'wb')
pickle.dump(automl, modelfile)
modelfile.close()
files.append(model_name)
model_dir = os.getcwd()
return model_name, model_dir, files
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss, accuracy_score
from gama import GamaClassifier
if __name__ == '__main__':
X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
automl = GamaClassifier(max_total_time=180,
keep_analysis_log=None,
n_jobs=1)
print("Starting `fit` which will take roughly 3 minutes.")
automl.fit(X_train, y_train)
label_predictions = automl.predict(X_test)
probability_predictions = automl.predict_proba(X_test)
print('accuracy:', accuracy_score(y_test, label_predictions))
print('log loss:', log_loss(y_test, probability_predictions))
def _test_dataset_problem(
data,
metric: str,
arff: bool = False,
y_type: Type = pd.DataFrame,
search: BaseSearch = AsyncEA(),
missing_values: bool = False,
max_time: int = 60,
):
"""
:param data:
:param metric:
:param arff:
:param y_type: pd.DataFrame, pd.Series, np.ndarray or str
:return:
"""
gama = GamaClassifier(
random_state=0,
max_total_time=max_time,
scoring=metric,
search=search,
n_jobs=1,
post_processing=EnsemblePostProcessing(ensemble_size=5),
store="nothing",
)
if arff:
train_path = f"tests/data/{data['name']}_train.arff"
test_path = f"tests/data/{data['name']}_test.arff"
X, y = data["load"](return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
y_test = [str(val) for val in y_test]
with Stopwatch() as sw:
gama.fit_from_file(train_path, target_column=data["target"])
class_predictions = gama.predict_from_file(
test_path, target_column=data["target"])
class_probabilities = gama.predict_proba_from_file(
test_path, target_column=data["target"])
gama_score = gama.score_from_file(test_path)
else:
X, y = data["load"](return_X_y=True)
if y_type == str:
databunch = data["load"]()
y = np.asarray(
[databunch.target_names[c_i] for c_i in databunch.target])
if y_type in [pd.Series, pd.DataFrame]:
y = y_type(y)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
if missing_values:
X_train[1:300:2, 0] = X_train[2:300:5, 1] = float("NaN")
X_test[1:100:2, 0] = X_test[2:100:5, 1] = float("NaN")
with Stopwatch() as sw:
gama.fit(X_train, y_train)
class_predictions = gama.predict(X_test)
class_probabilities = gama.predict_proba(X_test)
gama_score = gama.score(X_test, y_test)
assert (60 * FIT_TIME_MARGIN >
sw.elapsed_time), "fit must stay within 110% of allotted time."
assert isinstance(class_predictions,
np.ndarray), "predictions should be numpy arrays."
assert (
data["test_size"],
) == class_predictions.shape, "predict should return (N,) shaped array."
accuracy = accuracy_score(y_test, class_predictions)
# Majority classifier on this split achieves 0.6293706293706294
print(data["name"], metric, "accuracy:", accuracy)
assert (data["base_accuracy"] <= accuracy
), "predictions should be at least as good as majority class."
assert isinstance(
class_probabilities,
np.ndarray), "probability predictions should be numpy arrays."
assert (data["test_size"],
data["n_classes"]) == class_probabilities.shape, (
"predict_proba should return"
" (N,K) shaped array.")
# Majority classifier on this split achieves 12.80138131184662
logloss = log_loss(y_test, class_probabilities)
print(data["name"], metric, "log-loss:", logloss)
assert (data["base_log_loss"] >= logloss
), "predictions should be at least as good as majority class."
score_to_match = logloss if metric == "neg_log_loss" else accuracy
assert score_to_match == pytest.approx(gama_score)
gama.cleanup("all")
return gama
# In[6]:
#Initialization
cls = GamaClassifier(max_total_time=3600,
keep_analysis_log=None,
n_jobs=1,
scoring='accuracy',
post_processing_method=EnsemblePostProcessing())
X = B[0].iloc[:, 0:-1]
y = B[0].iloc[:, -1]
print("Starting `fit`")
cls.fit(X, y)
anytime_model = cls
#Prequential evaluation
for i in range(1, n):
#Test on next batch for accuracy
X = B[i].iloc[:, 0:-1]
y = B[i].iloc[:, -1]
y_hat = cls.predict(X)
accuracy = sklearn.metrics.accuracy_score(y, y_hat)
print("Test batch %d - Test score %f\n" % (i, accuracy))
# In[ ]:
X_train = B[start - 1].iloc[:, 0:-1]
y_train = B[start - 1].iloc[:, -1]
print("Starting to `fit`")
cls.fit(X_train, y_train, warm_start=True)
anytime_model = cls
#Prequential evaluation
for i in range(start, n):
#Test on next batch for accuracy
X_test = B[i].iloc[:, 0:-1]
y_test = B[i].iloc[:, -1]
y_hat = cls.predict(X_test)
b_acc = sklearn.metrics.balanced_accuracy_score(
y_test, y_hat) #equivalent to ROC_AUC in binary case
acc = sklearn.metrics.accuracy_score(y_test, y_hat)
print("Test batch %d - Balanced accuracy %f - Accuracy %f\n" %
(i, b_acc, acc))
#Check for drift
drift_in_batch = 0
for j in range(0, len(B[i])):
drift_detector.add_element(int(not y_test.iloc[j] == y_hat[j]))
if drift_detector.detected_change():
print('Drift in performance detected at sample {}'.format(j))
drift_in_batch = 1
def _test_dataset_problem(data,
metric: str,
arff: bool = False,
y_type: Type = pd.DataFrame,
search: BaseSearch = AsyncEA(),
missing_values: bool = False,
max_time: int = 60):
"""
:param data:
:param metric:
:param arff:
:param y_type: pd.DataFrame, pd.Series, np.ndarray or str
:return:
"""
gama = GamaClassifier(
random_state=0,
max_total_time=max_time,
scoring=metric,
search_method=search,
n_jobs=1,
post_processing_method=EnsemblePostProcessing(ensemble_size=5))
if arff:
train_path = 'tests/data/{}_train.arff'.format(data['name'])
test_path = 'tests/data/{}_test.arff'.format(data['name'])
X, y = data['load'](return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
y_test = [str(val) for val in y_test]
with Stopwatch() as sw:
gama.fit_arff(train_path, target_column=data['target'])
class_predictions = gama.predict_arff(test_path,
target_column=data['target'])
class_probabilities = gama.predict_proba_arff(
test_path, target_column=data['target'])
gama_score = gama.score_arff(test_path)
else:
X, y = data['load'](return_X_y=True)
if y_type == str:
databunch = data['load']()
y = np.asarray(
[databunch.target_names[c_i] for c_i in databunch.target])
if y_type in [pd.Series, pd.DataFrame]:
y = y_type(y)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
if missing_values:
X_train[1:300:2, 0] = X_train[2:300:5, 1] = float("NaN")
X_test[1:100:2, 0] = X_test[2:100:5, 1] = float("NaN")
with Stopwatch() as sw:
gama.fit(X_train, y_train)
class_predictions = gama.predict(X_test)
class_probabilities = gama.predict_proba(X_test)
gama_score = gama.score(X_test, y_test)
assert 60 * FIT_TIME_MARGIN > sw.elapsed_time, 'fit must stay within 110% of allotted time.'
assert isinstance(class_predictions,
np.ndarray), 'predictions should be numpy arrays.'
assert (
data['test_size'],
) == class_predictions.shape, 'predict should return (N,) shaped array.'
accuracy = accuracy_score(y_test, class_predictions)
# Majority classifier on this split achieves 0.6293706293706294
print(data['name'], metric, 'accuracy:', accuracy)
assert data[
'base_accuracy'] <= accuracy, 'predictions should be at least as good as majority class.'
assert isinstance(
class_probabilities,
np.ndarray), 'probability predictions should be numpy arrays.'
assert (data['test_size'],
data['n_classes']) == class_probabilities.shape, (
'predict_proba should return'
' (N,K) shaped array.')
# Majority classifier on this split achieves 12.80138131184662
logloss = log_loss(y_test, class_probabilities)
print(data['name'], metric, 'log-loss:', logloss)
assert data[
'base_log_loss'] >= logloss, 'predictions should be at least as good as majority class.'
score_to_match = logloss if metric == 'log_loss' else accuracy
assert score_to_match == pytest.approx(gama_score)