def test_fit_and_predict(self):
metric = Metric({"name": "logloss"})
automl = AutoML(
total_time_limit=5,
algorithms=["Xgboost"],
start_random_models=5,
hill_climbing_steps=0,
seed=13,
)
automl.fit(self.X, self.y)
y_predicted = automl.predict(self.X)["p_1"]
self.assertTrue(y_predicted is not None)
loss = metric(self.y, y_predicted)
self.assertTrue(loss < 0.7)
params = automl.to_json()
automl2 = AutoML()
automl2.from_json(params)
y_predicted2 = automl2.predict(self.X)["p_1"]
self.assertTrue(y_predicted2 is not None)
loss2 = metric(self.y, y_predicted2)
self.assertTrue(loss2 < 0.7)
assert_almost_equal(automl._threshold, automl2._threshold)
def test_fit_and_predict(self):
seed = 1709
df = pd.read_csv(
"./tests/data/housing_regression_missing_values_missing_target.csv"
)
print(df.columns)
x_cols = [c for c in df.columns if c != "MEDV"]
X = df[x_cols]
y = df["MEDV"]
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
X, y, test_size=0.3, random_state=seed)
automl = AutoML(
total_time_limit=10,
algorithms=["Xgboost"
], # ["LightGBM", "RF", "NN", "CatBoost", "Xgboost"],
start_random_models=1,
hill_climbing_steps=0,
top_models_to_improve=0,
train_ensemble=True,
verbose=True,
)
automl.fit(X_train, y_train)
response = automl.predict(X_test) # ["p_1"]
print("Response", response)
def test_fit_and_predict(self):
seed = 1706 + 1
for dataset_id in [31]: # 720 # 31,44,737
df = pd.read_csv("./tests/data/data/{0}.csv".format(dataset_id))
x_cols = [c for c in df.columns if c != "target"]
X = df[x_cols]
y = df["target"]
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
X, y, test_size=0.3, random_state=seed)
automl = AutoML(
total_time_limit=60 * 6000,
algorithms=["LightGBM", "RF", "NN", "CatBoost", "Xgboost"],
start_random_models=10,
hill_climbing_steps=3,
top_models_to_improve=3,
train_ensemble=True,
verbose=True,
)
automl.fit(X_train, y_train)
response = automl.predict(X_test)
# Compute the logloss on test dataset
ll = log_loss(y_test, response)
print("(*) Dataset id {} logloss {}".format(dataset_id, ll))
for i, m in enumerate(automl._models):
response = m.predict(X_test)
ll = log_loss(y_test, response)
print("{}) Dataset id {} logloss {}".format(i, dataset_id, ll))
def run(dataset, config):
log.info("\n**** mljar-supervised ****\n")
column_names, _ = zip(*dataset.columns)
column_types = dict(dataset.columns)
X_train = pd.DataFrame(dataset.train.X,
columns=column_names).astype(column_types,
copy=False)
X_test = pd.DataFrame(dataset.test.X,
columns=column_names).astype(column_types,
copy=False)
y_train = dataset.train.y.flatten()
y_test = dataset.test.y.flatten()
problem_mapping = dict(
binary="binary_classification",
multiclass="multiclass_classification",
regression="regression",
)
is_classification = config.type == "classification"
ml_task = problem_mapping.get(
dataset.problem_type
) # if None the AutoML will guess about the ML task
results_path = output_subdir("results", config)
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith("_")
}
automl = AutoML(results_path=results_path,
total_time_limit=config.max_runtime_seconds,
seed=config.seed,
ml_task=ml_task,
**training_params)
with Timer() as training:
automl.fit(X_train, y_train)
preds = automl.predict(X_test)
predictions, probabilities = None, None
if is_classification:
predictions = preds["label"].values
probabilities = preds[preds.columns[:-1]].values
else:
predictions = preds["prediction"].values
# clean the results
if not config.framework_params.get("_save_artifacts", False):
shutil.rmtree(results_path, ignore_errors=True)
return result(
output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
models_count=len(automl._models),
training_duration=training.duration,
)
def run(self):
from supervised.automl import AutoML
dataset = Dataset(self.specification['input'])
dataframe = dataset.get_dataframe().dropna()
X = self.specification['problem']['predictors']
y = self.specification['problem']['targets'][0]
stimulus, preprocessor = preprocess(dataframe, self.specification)
if self.specification.get('timeBoundSearch'):
self.system_params['total_time_limit'] = self.specification[
'timeBoundSearch']
if self.specification.get('timeBoundRun'):
self.system_params['learner_time_limit'] = self.specification[
'timeBoundRun']
automl = AutoML(**self.system_params)
# mljar seems kind of fragile?
stimulus = pandas.DataFrame(stimulus)
stimulus.columns = [str(i).strip() for i in stimulus.columns]
automl.fit(stimulus, dataframe[y])
for model_mljar in sorted(automl._models,
key=lambda m: m.get_final_loss())[:4]:
model = ModelSklearn(
model_mljar,
system='mljar-supervised',
search_id=self.search_id,
predictors=X,
targets=[y],
preprocess=preprocessor,
task=self.specification['problem']['taskType'])
model.save()
from tworaven_apps.solver_interfaces.tasks import FOUND_MODEL_CALLBACKS
FOUND_MODEL_CALLBACKS[self.callback_found](
model, **(self.callback_arguments or {}))
return {
KEY_SUCCESS: True,
KEY_MESSAGE: 'search complete',
KEY_DATA: {
'search_id': self.search_id,
'system': 'mljar-supervised'
}
}
def test_fit_optimize_auc(self):
automl = AutoML(
total_time_limit=5,
algorithms=["Xgboost"],
start_random_models=2,
hill_climbing_steps=0,
optimize_metric="auc",
seed=16,
)
automl.fit(self.X, self.y)
ldb = automl.get_leaderboard()
self.assertEqual(ldb["metric_type"][0], "auc")
self.assertEqual(np.sum(ldb["metric_value"] > 0.5),
ldb.shape[0]) # all better than 0.5 AUC
def test_predict_labels(self):
# 3.csv') #
df = pd.read_csv(
'tests/data/adult_missing_values_missing_target_500rows.csv')
X = df[df.columns[:-1]]
y = df[df.columns[-1]]
automl = AutoML(total_time_limit=15,
algorithms=["Xgboost"],
start_random_models=5,
hill_climbing_steps=0,
train_ensemble=True)
automl.fit(X, y)
y_predicted = automl.predict(X)
self.assertTrue('A' in np.unique(y_predicted['label']))
self.assertTrue('B' in np.unique(y_predicted['label']))
def train_titanic(train_data):
train_df = pd.read_csv(train_data)
# test_df = pd.read_csv(test_data)
# feature_cols = train_df.drop(['Survived', 'PassengerId', 'Name'], axis=1).columns
feature_cols = train_df.columns[2:]
target_cols = 'Survived'
X_train, X_test, y_train, y_test = train_test_split(train_df[feature_cols],
train_df[target_cols],
test_size=0.25)
automl = AutoML(results_path="AutoML_titanic")
automl.fit(X_train, y_train)
predictions = automl.predict(X_test)
print(f"Accuracy: {accuracy_score(y_test, predictions) * 100.0:.2f}%")
def test_predict_labels(self):
automl = AutoML(
total_time_limit=15,
algorithms=["Xgboost"],
start_random_models=5,
hill_climbing_steps=0,
train_ensemble=True,
seed=15,
)
automl.fit(self.X, self.y)
ldb = automl.get_leaderboard()
self.assertEqual(ldb.shape[0], len(automl._models))
for col in [
"uid", "model_type", "metric_type", "metric_value",
"train_time"
]:
self.assertTrue(col in ldb.columns)
def test_reproduce_fit(self):
metric = Metric({"name": "logloss"})
losses = []
for i in range(2):
automl = AutoML(
total_time_limit=
10000, # the time limit should be big enough too not interrupt the training
algorithms=["Xgboost"],
start_random_models=2,
hill_climbing_steps=1,
train_ensemble=True,
verbose=True,
seed=12,
)
automl.fit(self.X, self.y)
y_predicted = automl.predict(self.X)["p_1"]
loss = metric(self.y, y_predicted)
losses += [loss]
assert_almost_equal(losses[0], losses[1], decimal=4)
def train_digits():
digits = load_digits()
X_train, X_test, y_train, y_test = train_test_split(pd.DataFrame(
digits.data),
digits.target,
stratify=digits.target,
test_size=0.25,
random_state=123)
# train models with AutoML
automl = AutoML(mode="Perform", results_path="AutoML_digits")
automl.fit(X_train, y_train)
# compute
predictions = automl.predict_all(X_test)
print(predictions.head())
print("Test accuracy:",
accuracy_score(y_test, predictions["label"].astype(int)))
plot_digits(X_test, predictions)
def test_fit_and_predict(self):
for dataset_id in [3, 24, 31, 38, 44, 179, 737, 720]:
df = pd.read_csv("./tests/data/{0}.csv".format(dataset_id))
x_cols = [c for c in df.columns if c != "target"]
X = df[x_cols]
y = df["target"]
for repeat in range(1):
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
X, y, test_size=0.3, random_state=1706 + repeat)
automl = AutoML(
total_time_limit=60 * 1, # 1h limit
algorithms=[
"Xgboost"
], # ["LightGBM", "CatBoost", "Xgboost", "RF", "NN"],
start_random_models=3,
hill_climbing_steps=1,
top_models_to_improve=1,
train_ensemble=True,
verbose=True,
)
automl.fit(X_train, y_train)
response = automl.predict(X_test)["p_1"]
labels = automl.predict(X_test)["label"]
# Compute the logloss on test dataset
ll = log_loss(y_test, response)
f1 = f1_score(y_test, labels)
print("iter: {}) id:{} logloss:{} f1:{} time:{}".format(
repeat, dataset_id, ll, f1, automl._fit_time))
with open("./result.txt", "a") as f_result:
f_result.write("{} {} {} {} {}\n".format(
repeat, dataset_id, ll, f1, automl._fit_time))
category=pd.core.common.SettingWithCopyWarning) # message="*ndarray*")
# df = pd.read_csv("tests/data/iris_classes_missing_values_missing_target.csv")
df = pd.read_csv("tests/data/iris_missing_values_missing_target.csv")
X = df[["feature_1", "feature_2", "feature_3", "feature_4"]]
y = df["class"]
automl = AutoML(
# results_path="AutoML_41",
# algorithms=["CatBoost"],
#algorithms=["Neural Network"],
# "Linear",
# "Xgboost",
# "Random Forest"
# ],
#total_time_limit=100,
#tuning_mode="Normal",
#explain_level=0,
mode="Perform")
# automl.set_advanced(start_random_models=1)
automl.fit(X, y)
predictions = automl.predict(X)
print(predictions.head())
print(predictions.tail())
print(X.shape)
print(predictions.shape)
if (len(ratedf) == 0):
continue
featvals_stoch.append("stoch20" + "(" + str(0.1 * (i + 1)) + ")")
ratedf = k1[(k1.stoch20 > i * 0.1) & (k1.stoch20 < (i + 1) * 0.1)]
featrate_stoch.append(len(ratedf[ratedf.rets_long == 1]) / (len(ratedf)))
k1 = k1.dropna()
feats = [
'close_diff', 'gap1', 'rsi5', 'rsi5_smoothed', 'gap', 'stoch20', 'stoch14',
'rsi14', 'rsi20', 'sine', 'bandpass', 'cci', 'decycle', 'quadlead',
'velacc', 'VIX_Close', 'VIX_Close_diff', 'h', 'rsi20_diff', 'rsi14_diff',
'stoch20_diff', 'res1', 'res2', 'res3', 'res4', 'res5'
]
feats1 = feats
xtrain = np.array(k1[feats1])
ytrain = np.array(k1.rets_long)
tr = RandomForestClassifier(n_estimators=550,
max_depth=6,
min_samples_split=10)
clf = tr
#clf = AdaBoostClassifier(base_estimator=tr,n_estimators=80,random_state=50,learning_rate=1.0)
automl.fit(xtrain, ytrain)
k2['predictions'] = automl.predict(np.array(k2[feats1]))
k2['rand_preds'] = [random.choice([0, 1]) for _ in range(len(k2))]
print("accs:")
print(len(k2[k2.predictions == k2.rets1]) / (len(k2)))
print("rets original:")
print(k2.rets.sum())
print("rets model")
print((k2.predictions * k2.rets).sum())
#y=y+.1
y = y.reshape(len(y), )
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.3)
#model = Sequential([
#Dense(input_shape=(x_train.shape[1],), units=1,kernel_initializer=tf.constant_initializer(1),
# bias_initializer=tf.constant_initializer(0),kernel_constraint=MinMaxNorm(min_value=-1,max_value=1)),
# tfpl.DistributionLambda(lambda t:tfd.Exponential(rate=t),
# convert_to_tensor_fn=tfd.Distribution.sample)
#])
#model.compile(loss=nll,
#optimizer=RMSprop(learning_rate=0.01))
#model.fit(x_train, y_train, epochs=1000, verbose=True);
#tr = RandomForestRegressor(n_estimators=50,max_depth=6,min_samples_split=10)
#clf=tr
#clf = AdaBoostClassifier(base_estimator=tr,n_estimators=80,random_state=50,learning_rate=1.0)
automl.fit(x_train, y_train)
print("MAP: ", mean_absolute_percentage_error(automl.predict(x_test), y_test))
dftt = pd.DataFrame()
dftt.loc[ind[0], cols[0]] = df[(df.doctype == 'SOW') & (df.qornot == 'Yes') &
(df.pages < sowmed)].tat.mean()
dftt.loc[ind[1], cols[0]] = df[(df.doctype == 'SOW') & (df.qornot == 'No') &
(df.pages < sowmed)].tat.mean()
dftt.loc[ind[2], cols[0]] = ttest(
df[(df.doctype == 'SOW') & (df.qornot == 'Yes') & (df.pages < sowmed)].tat,
df[(df.doctype == 'SOW') & (df.qornot == 'No') & (df.pages < sowmed)].tat)
dftt.loc[ind[0], cols[1]] = df[(df.doctype == 'SOW') & (df.qornot == 'Yes') &
(df.pages > sowmed)].tat.mean()
dftt.loc[ind[1], cols[1]] = df[(df.doctype == 'SOW') & (df.qornot == 'No') &
(df.pages > sowmed)].tat.mean()
dftt.loc[ind[2], cols[1]] = ttest(
def run(dataset, config):
log.info(f"\n**** mljar-supervised [v{supervised.__version__}] ****\n")
save_metadata(config, version=supervised.__version__)
# Mapping of benchmark metrics to MLJAR metrics
metrics_mapping = dict(auc='auc', logloss='logloss', rmse='rmse')
eval_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else "auto"
# Mapping of benchmark task to MLJAR ML task
problem_mapping = dict(
binary="binary_classification",
multiclass="multiclass_classification",
regression="regression",
)
ml_task = problem_mapping.get(
dataset.problem_type
) # if None the AutoML will guess about the ML task
is_classification = config.type == "classification"
results_path = output_subdir("results", config)
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith("_")
}
column_names, _ = zip(*dataset.columns)
column_types = dict(dataset.columns)
label = dataset.target.name
train = pd.DataFrame(dataset.train.data,
columns=column_names).astype(column_types, copy=False)
X_train = train.drop(columns=label)
y_train = train[label]
test = pd.DataFrame(dataset.test.data,
columns=column_names).astype(column_types, copy=False)
X_test = test.drop(columns=label)
y_test = test[label]
automl = AutoML(results_path=results_path,
total_time_limit=config.max_runtime_seconds,
random_state=config.seed,
ml_task=ml_task,
eval_metric=eval_metric,
**training_params)
with utils.Timer() as training:
automl.fit(X_train, y_train)
with utils.Timer() as predict:
preds = automl.predict_all(X_test)
predictions, probabilities = None, None
if is_classification:
predictions = preds["label"].values
cols = [f"prediction_{c}" for c in np.unique(y_train)]
probabilities = preds[cols].values
else:
predictions = preds["prediction"].values
# clean the results
if not config.framework_params.get("_save_artifacts", False):
shutil.rmtree(results_path, ignore_errors=True)
return result(output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
models_count=len(automl._models),
training_duration=training.duration,
predict_duration=predict.duration)
#
# automl.train(y="Hall_of_Fame", x=['At_bats', 'Runs'], training_frame=data, fold_column='TWORAVENS_FOLD_COLUMN')
#
# best_model = h2o.get_model(automl.leaderboard.as_data_frame()['model_id'][0])
#
# print(best_model.cross_validation_fold_assignment())
#
from supervised.automl import AutoML
import pandas as pd
dataframe = pd.read_csv(data_path)
dataframe = dataframe[dataframe['Hall_of_Fame'] != 2]
automl_mljar = AutoML(total_time_limit=30)
automl_mljar.fit(dataframe[['Runs', 'At_bats']], dataframe['Hall_of_Fame'])
mljar_model = automl_mljar._models[0]
mljar_model.train({
"train": {
"X": dataframe[['Runs', 'At_bats']],
"y": dataframe['Hall_of_Fame']
}
})
mljar_model.predict(dataframe[['Runs', 'At_bats']])
# import mlbox.model.classification
# import mlbox.model.regression
#
#
import pandas as pd
from supervised.automl import AutoML
import os
df = pd.read_csv(
"https://raw.githubusercontent.com/pplonski/datasets-for-start/master/adult/data.csv",
skipinitialspace=True,
)
X = df[df.columns[:-1]]
y = df["income"]
automl = AutoML()
# results_path = "AutoML_8",
# total_time_limit=5,
# start_random_models=1,
# hill_climbing_steps=0,
# top_models_to_improve=3,
# train_ensemble=True)
print(X)
print(y)
automl.fit(X, y)
automl.fit(s, )
print(X)
print(y)
predictions = automl.predict(X)
print(predictions.head())
# tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2)
# tpot.fit(under_sampled_X,under_sampled_Y)
# print(tpot.score(test_X,test_Y))
# tpot.export('tpot_model_churn.py')
#%%
from supervised.automl import AutoML
automl = AutoML(total_time_limit=180 * 60,
top_models_to_improve=3,
learner_time_limit=240,
algorithms=["Xgboost", "RF", "LightGBM"],
start_random_models=10,
hill_climbing_steps=4)
automl.fit(under_sampled_X, under_sampled_Y)
predictions = automl.predict(test_X)
#%%
accuracy = accuracy_score(test_Y.values, predictions['label'].values)
print("\n Classification report : \n",
classification_report(test_Y, predictions['label'].values))
print("Accuracy Score : ", accuracy)
#confusion matrix
conf_matrix = confusion_matrix(test_Y, predictions['label'].values)
#roc_auc_score
model_roc_auc = roc_auc_score(test_Y, predictions['label'].values)
print("Area under curve : ", model_roc_auc, "\n")
train_ensemble=False,
golden_features=False,
features_selection=False,
ml_task="regression",
)
nn = AutoML(
algorithms=["Neural Network"],
mode="Perform",
explain_level=0,
train_ensemble=False,
golden_features=False,
features_selection=False,
ml_task="regression",
)
mlp.fit(train_X, train_y)
mlp_time = np.round(time.time() - mlp._start_time, 2)
nn.fit(train_X, train_y)
nn_time = np.round(time.time() - nn._start_time, 2)
mlp_mse = mean_squared_error(test_y, mlp.predict(test_X))
nn_mse = mean_squared_error(test_y, nn.predict(test_X))
print(dataset, X.shape, np.unique(y), mlp_mse, nn_mse)
results += [{
"dataset": dataset,
"nrows": X.shape[0],
"ncols": X.shape[1],
"mlp_mse": mlp_mse,
"nn_mse": nn_mse,
import pandas as pd
# scikit learn utilites
from sklearn.datasets import load_digits
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
# mljar-supervised package
from supervised.automl import AutoML
# Load the data
digits = load_digits()
X_train, X_test, y_train, y_test = train_test_split(pd.DataFrame(digits.data),
digits.target,
stratify=digits.target,
test_size=0.25)
# train models
automl = AutoML(mode="Perform")
automl.fit(X_train, y_train)
# compute the accuracy on test data
predictions = automl.predict(X_test)
print(predictions.head())
print("Test accuracy:", accuracy_score(y_test,
predictions["label"].astype(int)))
def run(dataset, config):
log.info(f"\n**** mljar-supervised [v{supervised.__version__}] ****\n")
# Mapping of benchmark metrics to MLJAR metrics
metrics_mapping = dict(auc='auc', logloss='logloss', rmse='rmse')
eval_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else "auto"
# Mapping of benchmark task to MLJAR ML task
problem_mapping = dict(
binary="binary_classification",
multiclass="multiclass_classification",
regression="regression",
)
ml_task = problem_mapping.get(
dataset.problem_type
) # if None the AutoML will guess about the ML task
is_classification = config.type == "classification"
results_path = output_subdir("results", config)
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith("_")
}
X_train, y_train = dataset.train.X, dataset.train.y.squeeze()
X_test, y_test = dataset.test.X, dataset.test.y.squeeze()
automl = AutoML(results_path=results_path,
total_time_limit=config.max_runtime_seconds,
random_state=config.seed,
ml_task=ml_task,
eval_metric=eval_metric,
**training_params)
with Timer() as training:
automl.fit(X_train, y_train)
with Timer() as predict:
preds = automl.predict_all(X_test)
predictions, probabilities, probabilities_labels = None, None, None
if is_classification:
# preds is a dataframe with columns ["prediction_LABEL", .., "label"]
if y_train.dtype == bool and preds["label"].dtype == int:
# boolean target produces integer predictions for mljar-supervised <= 0.10.6
# https://github.com/mljar/mljar-supervised/issues/442
preds = preds.rename(
{
"prediction_0": "False",
"prediction_1": "True"
}, axis=1)
preds["label"] = preds["label"].astype(bool)
else:
preds.columns = [
c.replace("prediction_", "", 1) for c in preds.columns
]
predictions = preds["label"].values
probabilities_labels = list(preds.columns)[:-1]
probabilities = preds[probabilities_labels].values
else:
predictions = preds["prediction"].values
# clean the results
if not config.framework_params.get("_save_artifacts", False):
shutil.rmtree(results_path, ignore_errors=True)
return result(output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
probabilities_labels=probabilities_labels,
models_count=len(automl._models),
training_duration=training.duration,
predict_duration=predict.duration)