def train_gama(X_train, X_test, y_train, y_test, mtype, common_name_model,
problemtype, classes, default_featurenames, transform_model,
settings, model_session):
model_name = common_name_model + '.pickle'
files = list()
if mtype in ['c']:
automl = GamaClassifier(max_total_time=180, keep_analysis_log=None)
print(
"Starting GAMA `fit` - usually takes around 3 minutes but can take longer for large datasets"
)
automl.fit(X_train, y_train)
label_predictions = automl.predict(X_test)
probability_predictions = automl.predict_proba(X_test)
accuracy = accuracy_score(y_test, label_predictions)
log_loss_pred = log_loss(y_test, probability_predictions)
log_loss_score = automl.score(X_test, y_test)
print('accuracy:', accuracy)
print('log loss pred:', log_loss_pred)
print('log_loss_score', log_loss_score)
elif mtype in ['regression', 'r']:
automl = GamaRegressor(max_total_time=180,
keep_analysis_log=None,
n_jobs=1)
print(
"Starting GAMA `fit` - usually takes around 3 minutes but can take longer for large datasets"
)
automl.fit(X_train, y_train)
predictions = automl.predict(X_test)
mse_error = mean_squared_error(y_test, predictions)
print("MSE:", mse_error)
# SAVE ML MODEL
modelfile = open(model_name, 'wb')
pickle.dump(automl, modelfile)
modelfile.close()
files.append(model_name)
model_dir = os.getcwd()
return model_name, model_dir, files
def gama_runs(datasets, path, task):
''' Executes Gama optimization for different OpenML datasets and stores the
log files in a specified path.
Parameters:
-----------
datasets: list
Contains datasets that are going to be optimized using Gama.
path: string
Contains the path to the directory in where the files are logged.
task: string
Contains learning task to specify the GAMA optimization (either classi-
fication or regression).
Returns:
--------
string
Contains a confirmation that the optimization process has finished.
'''
executed = executed_datasets(path)
for dataset_id in datasets:
if dataset_id not in executed:
try:
ds = oml.datasets.get_dataset(dataset_id, download_data=False)
X, y, categorical_indicator, attribute_names = ds.get_data(
dataset_format='DataFrame',
target=ds.default_target_attribute)
categorical, numeric, string = category_numeric_or_string(
X, categorical_indicator)
X, y = impute(X, y, categorical, numeric, string, "median")
for k in [1, 2, 5, 10, 25]:
log_k = ''
if k == 1:
log_k = 'a'
elif k == 2:
log_k = 'b'
elif k == 5:
log_k = 'c'
elif k == 10:
log_k = 'd'
else:
log_k = 'e'
X_adj, y_adj = onehot_or_targ(X, y, categorical, k)
if task.lower() == "classification":
gama = GamaClassifier(
n_jobs=-1,
max_total_time=600,
scoring='accuracy',
keep_analysis_log='{}{}{}.log'.format(
path, log_k, dataset_id))
elif task.lower() == "regression":
gama = GamaRegressor(
n_jobs=-1,
max_total_time=600,
scoring='r2',
keep_analysis_log='{}{}{}.log'.format(
path, log_k, dataset_id))
else:
return "Please select classification or regression as learning task!"
gama.fit(X_adj, y_adj)
except:
pass
return "Gama has finished running optimization."
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss, accuracy_score
from gama import GamaClassifier
if __name__ == '__main__':
X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
automl = GamaClassifier(max_total_time=180,
keep_analysis_log=None,
n_jobs=1)
print("Starting `fit` which will take roughly 3 minutes.")
automl.fit(X_train, y_train)
label_predictions = automl.predict(X_test)
probability_predictions = automl.predict_proba(X_test)
print('accuracy:', accuracy_score(y_test, label_predictions))
print('log loss:', log_loss(y_test, probability_predictions))
def _test_dataset_problem(
data,
metric: str,
arff: bool = False,
y_type: Type = pd.DataFrame,
search: BaseSearch = AsyncEA(),
missing_values: bool = False,
max_time: int = 60,
):
"""
:param data:
:param metric:
:param arff:
:param y_type: pd.DataFrame, pd.Series, np.ndarray or str
:return:
"""
gama = GamaClassifier(
random_state=0,
max_total_time=max_time,
scoring=metric,
search=search,
n_jobs=1,
post_processing=EnsemblePostProcessing(ensemble_size=5),
store="nothing",
)
if arff:
train_path = f"tests/data/{data['name']}_train.arff"
test_path = f"tests/data/{data['name']}_test.arff"
X, y = data["load"](return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
y_test = [str(val) for val in y_test]
with Stopwatch() as sw:
gama.fit_from_file(train_path, target_column=data["target"])
class_predictions = gama.predict_from_file(
test_path, target_column=data["target"])
class_probabilities = gama.predict_proba_from_file(
test_path, target_column=data["target"])
gama_score = gama.score_from_file(test_path)
else:
X, y = data["load"](return_X_y=True)
if y_type == str:
databunch = data["load"]()
y = np.asarray(
[databunch.target_names[c_i] for c_i in databunch.target])
if y_type in [pd.Series, pd.DataFrame]:
y = y_type(y)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
if missing_values:
X_train[1:300:2, 0] = X_train[2:300:5, 1] = float("NaN")
X_test[1:100:2, 0] = X_test[2:100:5, 1] = float("NaN")
with Stopwatch() as sw:
gama.fit(X_train, y_train)
class_predictions = gama.predict(X_test)
class_probabilities = gama.predict_proba(X_test)
gama_score = gama.score(X_test, y_test)
assert (60 * FIT_TIME_MARGIN >
sw.elapsed_time), "fit must stay within 110% of allotted time."
assert isinstance(class_predictions,
np.ndarray), "predictions should be numpy arrays."
assert (
data["test_size"],
) == class_predictions.shape, "predict should return (N,) shaped array."
accuracy = accuracy_score(y_test, class_predictions)
# Majority classifier on this split achieves 0.6293706293706294
print(data["name"], metric, "accuracy:", accuracy)
assert (data["base_accuracy"] <= accuracy
), "predictions should be at least as good as majority class."
assert isinstance(
class_probabilities,
np.ndarray), "probability predictions should be numpy arrays."
assert (data["test_size"],
data["n_classes"]) == class_probabilities.shape, (
"predict_proba should return"
" (N,K) shaped array.")
# Majority classifier on this split achieves 12.80138131184662
logloss = log_loss(y_test, class_probabilities)
print(data["name"], metric, "log-loss:", logloss)
assert (data["base_log_loss"] >= logloss
), "predictions should be at least as good as majority class."
score_to_match = logloss if metric == "neg_log_loss" else accuracy
assert score_to_match == pytest.approx(gama_score)
gama.cleanup("all")
return gama
# In[6]:
#Initialization
cls = GamaClassifier(max_total_time=3600,
keep_analysis_log=None,
n_jobs=1,
scoring='accuracy',
post_processing_method=EnsemblePostProcessing())
X = B[0].iloc[:, 0:-1]
y = B[0].iloc[:, -1]
print("Starting `fit`")
cls.fit(X, y)
anytime_model = cls
#Prequential evaluation
for i in range(1, n):
#Test on next batch for accuracy
X = B[i].iloc[:, 0:-1]
y = B[i].iloc[:, -1]
y_hat = cls.predict(X)
accuracy = sklearn.metrics.accuracy_score(y, y_hat)
print("Test batch %d - Test score %f\n" % (i, accuracy))
# In[ ]:
#Initialization
cls = GamaClassifier(max_total_time=3600,
keep_analysis_log=None,
n_jobs=1,
scoring='log_loss',
post_processing_method=EnsemblePostProcessing())
#drift_detector = ADWIN()
drift_detector = EDDM()
start = 1
X_train = B[start - 1].iloc[:, 0:-1]
y_train = B[start - 1].iloc[:, -1]
print("Starting to `fit`")
cls.fit(X_train, y_train, warm_start=True)
anytime_model = cls
#Prequential evaluation
for i in range(start, n):
#Test on next batch for accuracy
X_test = B[i].iloc[:, 0:-1]
y_test = B[i].iloc[:, -1]
y_hat = cls.predict(X_test)
b_acc = sklearn.metrics.balanced_accuracy_score(
y_test, y_hat) #equivalent to ROC_AUC in binary case
acc = sklearn.metrics.accuracy_score(y_test, y_hat)
cls = GamaClassifier(max_total_time=3600,
keep_analysis_log=None,
n_jobs=1,
scoring='log_loss',
post_processing_method=EnsemblePostProcessing(),
config=limited_config)
drift_detector = EDDM()
start = 1
X_train = B[start - 1].iloc[:, 0:-1]
y_train = B[start - 1].iloc[:, -1]
print("Starting to `fit`")
cls.fit(X_train, y_train)
anytime_model = cls
#Prequential evaluation
for i in range(start, n):
#Test on next batch for accuracy
X_test = B[i].iloc[:, 0:-1]
y_test = B[i].iloc[:, -1]
y_hat = cls.predict(X_test)
b_acc = sklearn.metrics.balanced_accuracy_score(
y_test, y_hat) #equivalent to ROC_AUC in binary case
acc = sklearn.metrics.accuracy_score(y_test, y_hat)
def _test_dataset_problem(data,
metric: str,
arff: bool = False,
y_type: Type = pd.DataFrame,
search: BaseSearch = AsyncEA(),
missing_values: bool = False,
max_time: int = 60):
"""
:param data:
:param metric:
:param arff:
:param y_type: pd.DataFrame, pd.Series, np.ndarray or str
:return:
"""
gama = GamaClassifier(
random_state=0,
max_total_time=max_time,
scoring=metric,
search_method=search,
n_jobs=1,
post_processing_method=EnsemblePostProcessing(ensemble_size=5))
if arff:
train_path = 'tests/data/{}_train.arff'.format(data['name'])
test_path = 'tests/data/{}_test.arff'.format(data['name'])
X, y = data['load'](return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
y_test = [str(val) for val in y_test]
with Stopwatch() as sw:
gama.fit_arff(train_path, target_column=data['target'])
class_predictions = gama.predict_arff(test_path,
target_column=data['target'])
class_probabilities = gama.predict_proba_arff(
test_path, target_column=data['target'])
gama_score = gama.score_arff(test_path)
else:
X, y = data['load'](return_X_y=True)
if y_type == str:
databunch = data['load']()
y = np.asarray(
[databunch.target_names[c_i] for c_i in databunch.target])
if y_type in [pd.Series, pd.DataFrame]:
y = y_type(y)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
random_state=0)
if missing_values:
X_train[1:300:2, 0] = X_train[2:300:5, 1] = float("NaN")
X_test[1:100:2, 0] = X_test[2:100:5, 1] = float("NaN")
with Stopwatch() as sw:
gama.fit(X_train, y_train)
class_predictions = gama.predict(X_test)
class_probabilities = gama.predict_proba(X_test)
gama_score = gama.score(X_test, y_test)
assert 60 * FIT_TIME_MARGIN > sw.elapsed_time, 'fit must stay within 110% of allotted time.'
assert isinstance(class_predictions,
np.ndarray), 'predictions should be numpy arrays.'
assert (
data['test_size'],
) == class_predictions.shape, 'predict should return (N,) shaped array.'
accuracy = accuracy_score(y_test, class_predictions)
# Majority classifier on this split achieves 0.6293706293706294
print(data['name'], metric, 'accuracy:', accuracy)
assert data[
'base_accuracy'] <= accuracy, 'predictions should be at least as good as majority class.'
assert isinstance(
class_probabilities,
np.ndarray), 'probability predictions should be numpy arrays.'
assert (data['test_size'],
data['n_classes']) == class_probabilities.shape, (
'predict_proba should return'
' (N,K) shaped array.')
# Majority classifier on this split achieves 12.80138131184662
logloss = log_loss(y_test, class_probabilities)
print(data['name'], metric, 'log-loss:', logloss)
assert data[
'base_log_loss'] >= logloss, 'predictions should be at least as good as majority class.'
score_to_match = logloss if metric == 'log_loss' else accuracy
assert score_to_match == pytest.approx(gama_score)