def save_artifacts(predictor, leaderboard, config):
artifacts = config.framework_params.get('_save_artifacts', ['leaderboard'])
try:
models_dir = output_subdir("models", config)
shutil.rmtree(os.path.join(models_dir, "utils"), ignore_errors=True)
if 'leaderboard' in artifacts:
save_pd.save(path=os.path.join(models_dir, "leaderboard.csv"),
df=leaderboard)
if 'info' in artifacts:
ag_info = predictor.info()
info_dir = output_subdir("info", config)
save_pkl.save(path=os.path.join(info_dir, "info.pkl"),
object=ag_info)
if 'models' in artifacts:
utils.zip_path(models_dir, os.path.join(models_dir, "models.zip"))
def delete(path, isdir):
if isdir:
shutil.rmtree(path, ignore_errors=True)
elif os.path.splitext(path)[1] == '.pkl':
os.remove(path)
utils.walk_apply(models_dir, delete, max_depth=0)
except Exception:
log.warning("Error when saving artifacts.", exc_info=True)
def save_artifacts(estimator, config, start, dataset):
try:
models_repr = estimator.show_models()
log.debug("Trained Ensemble:\n%s", models_repr)
artifacts = config.framework_params.get('_save_artifacts', [])
if 'models' in artifacts:
config_file = os.path.join(output_subdir('models', config),
'configspace.pkl')
try:
with open(config_file, 'wb') as f:
cs = estimator.get_configuration_space(
dataset.train.X_enc, dataset.train.y_enc)
print(cs)
pickle.dump(cs, f)
except Exception as ex:
log.exception('Error: {}'.format(ex))
start_file = os.path.join(output_subdir('models', config),
'start.txt')
with open(start_file, 'w') as f:
f.write(str(start))
models_file = os.path.join(output_subdir('models', config),
'models.txt')
with open(models_file, 'w') as f:
f.write(models_repr)
for idx, automl in enumerate(estimator._automl):
runhistory_file = os.path.join(
output_subdir('models', config),
'runhistory_{}.json'.format(idx))
automl.runhistory_.save_json(runhistory_file)
except Exception:
log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(automl, dataset, config):
artifacts = config.framework_params.get('_save_artifacts', ['leaderboard'])
try:
lb = automl.leaderboard.as_data_frame()
log.debug("Leaderboard:\n%s", lb.to_string())
if 'leaderboard' in artifacts:
models_dir = output_subdir("models", config)
write_csv(lb, os.path.join(models_dir, "leaderboard.csv"))
if 'models' in artifacts:
models_dir = output_subdir("models", config)
all_models_se = next(
(mid for mid in lb['model_id']
if mid.startswith("StackedEnsemble_AllModels")), None)
mformat = 'mojo' if 'mojos' in artifacts else 'json'
if all_models_se and mformat == 'mojo':
save_model(all_models_se, dest_dir=models_dir, mformat=mformat)
else:
for mid in lb['model_id']:
save_model(mid, dest_dir=models_dir, mformat=mformat)
models_archive = os.path.join(models_dir, "models.zip")
zip_path(models_dir, models_archive)
def delete(path, isdir):
if path != models_archive and os.path.splitext(
path)[1] in ['.json', '.zip']:
os.remove(path)
walk_apply(models_dir, delete, max_depth=0)
if 'models_predictions' in artifacts:
predictions_dir = output_subdir("predictions", config)
test = h2o.get_frame(frame_name('test', config))
for mid in lb['model_id']:
model = h2o.get_model(mid)
save_predictions(model,
test,
dataset=dataset,
config=config,
predictions_file=os.path.join(
predictions_dir, mid, 'predictions.csv'),
preview=False)
zip_path(predictions_dir,
os.path.join(predictions_dir, "models_predictions.zip"))
def delete(path, isdir):
if isdir:
shutil.rmtree(path, ignore_errors=True)
walk_apply(predictions_dir, delete, max_depth=0)
if 'logs' in artifacts:
logs_dir = output_subdir("logs", config)
h2o.download_all_logs(dirname=logs_dir)
except Exception:
log.debug("Error when saving artifacts.", exc_info=True)
def run(dataset, config):
log.info("\n**** mljar-supervised ****\n")
column_names, _ = zip(*dataset.columns)
column_types = dict(dataset.columns)
X_train = pd.DataFrame(dataset.train.X,
columns=column_names).astype(column_types,
copy=False)
X_test = pd.DataFrame(dataset.test.X,
columns=column_names).astype(column_types,
copy=False)
y_train = dataset.train.y.flatten()
y_test = dataset.test.y.flatten()
problem_mapping = dict(
binary="binary_classification",
multiclass="multiclass_classification",
regression="regression",
)
is_classification = config.type == "classification"
ml_task = problem_mapping.get(
dataset.problem_type
) # if None the AutoML will guess about the ML task
results_path = output_subdir("results", config)
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith("_")
}
automl = AutoML(results_path=results_path,
total_time_limit=config.max_runtime_seconds,
seed=config.seed,
ml_task=ml_task,
**training_params)
with Timer() as training:
automl.fit(X_train, y_train)
preds = automl.predict(X_test)
predictions, probabilities = None, None
if is_classification:
predictions = preds["label"].values
probabilities = preds[preds.columns[:-1]].values
else:
predictions = preds["prediction"].values
# clean the results
if not config.framework_params.get("_save_artifacts", False):
shutil.rmtree(results_path, ignore_errors=True)
return result(
output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
models_count=len(automl._models),
training_duration=training.duration,
)
def run(dataset, config):
log.info(f"\n**** FLAML [v{__version__}] ****\n")
X_train, y_train = dataset.train.X, dataset.train.y.squeeze()
X_test, y_test = dataset.test.X, dataset.test.y.squeeze()
is_classification = config.type == 'classification'
time_budget = config.max_runtime_seconds
n_jobs = config.framework_params.get('_n_jobs', config.cores)
log.info("Running FLAML with {} number of cores".format(config.cores))
aml = AutoML()
# Mapping of benchmark metrics to flaml metrics
metrics_mapping = dict(
acc='accuracy',
auc='roc_auc',
f1='f1',
logloss='log_loss',
mae='mae',
mse='mse',
rmse='rmse',
r2='r2',
)
perf_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else 'auto'
if perf_metric is None:
log.warning("Performance metric %s not supported.", config.metric)
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
log_dir = output_subdir("logs", config)
flaml_log_file_name = os.path.join(log_dir, "flaml.log")
with Timer() as training:
aml.fit(X_train,
y_train,
metric=perf_metric,
task=config.type,
n_jobs=n_jobs,
log_file_name=flaml_log_file_name,
time_budget=time_budget,
**training_params)
with Timer() as predict:
predictions = aml.predict(X_test)
probabilities = aml.predict_proba(X_test) if is_classification else None
labels = aml.classes_ if is_classification else None
return result(
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
models_count=len(aml.config_history),
training_duration=training.duration,
predict_duration=predict.duration,
probabilities_labels=labels,
)
def save_artifacts(estimator: Union[GenensClassifier, GenensRegressor],
config):
try:
artifacts = config.framework_params.get('_save_artifacts', False)
if 'pickle_models' in artifacts:
models_dir = os.path.join(output_subdir('pickle_models', config))
# pickle top 3 best pipelines
for i, pipe in enumerate(estimator.get_best_pipelines()):
with open(models_dir + '/pipeline{}.pickle'.format(i),
'wb') as pickle_file:
pickle.dump(pipe, pickle_file, pickle.HIGHEST_PROTOCOL)
if 'models' in artifacts:
models_dir = os.path.join(output_subdir('models', config))
# top 3 individual fitness values
with open(models_dir + '/ind-fitness.txt', 'w+') as out_file:
best_inds = estimator.get_best_pipelines(as_individuals=True)
for i, ind in enumerate(best_inds):
out_file.write('Individual {}: Score {}\n'.format(
i, ind.fitness.values))
# individual tree
if 'models_png' in artifacts:
from genens.render.graph import create_graph
models_dir = os.path.join(output_subdir('models', config))
for i, ind in enumerate(best_inds):
create_graph(ind, models_dir + '/graph{}.png'.format(i))
if 'log' in artifacts:
log_dir = os.path.join(output_subdir('logs', config))
# write logbook string representation to output dir
with open(log_dir + '/logbook.txt', 'w+') as log_file:
log_file.write(estimator.logbook.__str__() + '\n')
# evolution plot
export_plot(estimator, log_dir + '/result.png')
except:
log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(chain, config):
try:
artifacts = config.framework_params.get('_save_artifacts', False)
if 'models' in artifacts:
models_file = os.path.join(output_subdir('models', config),
'model.json')
chain.save_chain(models_file)
except Exception:
log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(predictor, leaderboard, config):
artifacts = config.framework_params.get('_save_artifacts', ['leaderboard'])
try:
if 'leaderboard' in artifacts:
leaderboard_dir = output_subdir("leaderboard", config)
save_pd.save(path=os.path.join(leaderboard_dir, "leaderboard.csv"), df=leaderboard)
if 'info' in artifacts:
ag_info = predictor.info()
info_dir = output_subdir("info", config)
save_pkl.save(path=os.path.join(info_dir, "info.pkl"), object=ag_info)
if 'models' in artifacts:
shutil.rmtree(os.path.join(predictor.path, "utils"), ignore_errors=True)
models_dir = output_subdir("models", config)
utils.zip_path(predictor.path, os.path.join(models_dir, "models.zip"))
except Exception:
log.warning("Error when saving artifacts.", exc_info=True)
def save_artifacts(estimator, config):
try:
log.debug("All individuals :\n%s", list(estimator.evaluated_individuals_.items()))
models = estimator.pareto_front_fitted_pipelines_
hall_of_fame = list(zip(reversed(estimator._pareto_front.keys), estimator._pareto_front.items))
artifacts = config.framework_params.get('_save_artifacts', False)
if 'models' in artifacts:
models_file = os.path.join(output_subdir('models', config), 'models.txt')
with open(models_file, 'w') as f:
for m in hall_of_fame:
pprint.pprint(dict(
fitness=str(m[0]),
model=str(m[1]),
pipeline=models[str(m[1])],
), stream=f)
run_history_file = os.path.join(output_subdir('models', config), 'runhistory.pkl')
with open(run_history_file, 'wb') as f:
pickle.dump(estimator.run_history, f)
except Exception:
log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(automl, config):
try:
artifacts = config.framework_params.get('_save_artifacts', [])
models_dir = output_subdir("models", config)
if 'models' in artifacts:
with open(os.path.join(models_dir, 'automl.pickle'), 'wb') as f:
pickle.dump(automl, f)
except Exception:
log.warning("Error when saving artifacts.", exc_info=True)
def save_artifacts(estimator, config):
try:
models_repr = estimator.show_models()
log.debug("Trained Ensemble:\n%s", models_repr)
artifacts = config.framework_params.get('_save_artifacts', [])
if 'models' in artifacts:
models_file = os.path.join(output_subdir('models', config),
'models.txt')
with open(models_file, 'w') as f:
f.write(models_repr)
except Exception:
log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(estimator, config):
try:
models_repr = estimator.show_models()
log.debug("Trained Ensemble:\n%s", models_repr)
artifacts = config.framework_params.get('_save_artifacts', [])
if 'models' in artifacts:
models_file = os.path.join(output_subdir('models', config),
'models.txt')
with open(models_file, 'w') as f:
f.write(models_repr)
if 'debug_as_files' in artifacts or 'debug_as_zip' in artifacts:
print('Saving debug artifacts!')
debug_dir = output_subdir('debug', config)
ignore_extensions = [
'.npy', '.pcs', '.model', '.cv_model', '.ensemble', '.pkl'
]
tmp_directory = estimator.automl_._backend.temporary_directory
if 'debug_as_files' in artifacts:
def _copy(filename, **_):
dst = filename.replace(tmp_directory, debug_dir + '/')
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(filename, dst)
walk_apply(
tmp_directory,
_copy,
filter_=lambda path:
(os.path.splitext(path)[1] not in ignore_extensions and
not os.path.isdir(path)),
)
else:
zip_path(tmp_directory,
os.path.join(debug_dir, "artifacts.zip"),
filter_=lambda p: os.path.splitext(p)[1] not in
ignore_extensions)
except Exception as e:
log.debug("Error when saving artifacts= {e}.".format(e), exc_info=True)
def run(dataset, config):
jar_file = glob.glob("{here}/lib/mlplan/mlplan-cli*.jar".format(
here=os.path.dirname(__file__)))[0]
version = re.match(r".*/mlplan-cli-(.*).jar", jar_file)[1]
log.info(f"\n**** ML-Plan [v{version}] ****\n")
is_classification = config.type == 'classification'
# Mapping of benchmark metrics to Weka metrics
metrics_mapping = dict(acc='ERRORRATE',
auc='AUC',
logloss='LOGLOSS',
f1='F1',
r2='R2',
rmse='ROOT_MEAN_SQUARED_ERROR',
mse='MEAN_SQUARED_ERROR',
rmsle='ROOT_MEAN_SQUARED_LOGARITHM_ERROR',
mae='MEAN_ABSOLUTE_ERROR')
metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else None
if metric is None:
raise ValueError('Performance metric {} is not supported.'.format(
config.metric))
train_file = dataset.train.path
test_file = dataset.test.path
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
backend = config.framework_params.get('_backend', 'weka')
if backend == "weka":
mem_limit = str(max(config.max_mem_size_mb - 1024, 2048))
else:
mem_limit = str(
max(round((config.max_mem_size_mb - 1024) / config.cores), 2048))
mode = backend
if config.type == 'regression':
mode += '-regression'
log.info(
"Running ML-Plan with backend %s in mode %s and a maximum time of %ss on %s cores with %sMB for the JVM, optimizing %s.",
backend, mode, config.max_runtime_seconds, config.cores,
config.max_mem_size_mb, metric)
log.info("Environment: %s", os.environ)
predictions_file = os.path.join(output_subdir('mlplan_out', config),
'predictions.csv')
statistics_file = os.path.join(output_subdir('mlplan_out', config),
'statistics.json')
#tmp_dir = output_subdir('mlplan_tmp', config)
cmd_root = f"java -jar -Xmx{mem_limit}M {jar_file}"
with tempfile.TemporaryDirectory() as tmp_dir:
cmd_params = dict(
f='"{}"'.format(train_file),
p='"{}"'.format(test_file),
t=config.max_runtime_seconds,
ncpus=config.cores,
l=metric,
m=mode,
s=config.seed, # weka accepts only int16 as seeds
ooab=predictions_file,
os=statistics_file,
tmp=tmp_dir,
**training_params)
cmd = cmd_root + ''.join(
[" -{} {}".format(k, v) for k, v in cmd_params.items()])
with utils.Timer() as training:
utils.run_cmd(cmd, _live_output_=True)
with open(statistics_file, 'r') as f:
stats = json.load(f)
predictions = stats["predictions"]
truth = stats["truth"]
numEvals = stats["num_evaluations"]
# only for classification tasks we have probabilities available, thus check whether the json contains the respective fields
if "probabilities" in stats and "probabilities_labels" in stats:
probabilities = stats["probabilities"]
probabilities_labels = stats["probabilities_labels"]
else:
probabilities = []
probabilities_labels = []
return result(output_file=config.output_predictions_file,
predictions=predictions,
truth=truth,
probabilities=probabilities,
probabilities_labels=probabilities_labels,
target_is_encoded=is_classification,
models_count=numEvals,
training_duration=training.duration)
def run(dataset, config):
log.info(f"\n**** AutoGluon [v{__version__}] ****\n")
save_metadata(config, version=__version__)
metrics_mapping = dict(
acc=metrics.accuracy,
auc=metrics.roc_auc,
f1=metrics.f1,
logloss=metrics.log_loss,
mae=metrics.mean_absolute_error,
mse=metrics.mean_squared_error,
r2=metrics.r2,
# rmse=metrics.root_mean_squared_error, # metrics.root_mean_squared_error incorrectly registered in autogluon REGRESSION_METRICS
rmse=metrics.
mean_squared_error, # for now, we can let autogluon optimize training on mse: anyway we compute final score from predictions.
)
perf_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else None
if perf_metric is None:
# TODO: figure out if we are going to blindly pass metrics through, or if we use a strict mapping
log.warning("Performance metric %s not supported.", config.metric)
is_classification = config.type == 'classification'
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
column_names, _ = zip(*dataset.columns)
column_types = dict(dataset.columns)
train = pd.DataFrame(dataset.train.data,
columns=column_names).astype(column_types, copy=False)
label = dataset.target.name
print(f"Columns dtypes:\n{train.dtypes}")
output_dir = output_subdir("models", config)
with utils.Timer() as training:
predictor = task.fit(train_data=train,
label=label,
problem_type=dataset.problem_type,
output_directory=output_dir,
time_limits=config.max_runtime_seconds,
eval_metric=perf_metric.name,
**training_params)
test = pd.DataFrame(dataset.test.data,
columns=column_names).astype(column_types, copy=False)
X_test = test.drop(columns=label)
y_test = test[label]
with utils.Timer() as predict:
predictions = predictor.predict(X_test)
probabilities = predictor.predict_proba(
dataset=X_test, as_pandas=True,
as_multiclass=True) if is_classification else None
prob_labels = probabilities.columns.values.tolist(
) if probabilities is not None else None
leaderboard = predictor._learner.leaderboard(X_test, y_test, silent=True)
with pd.option_context('display.max_rows', None, 'display.max_columns',
None, 'display.width', 1000):
print(leaderboard)
save_artifacts(predictor, leaderboard, config)
num_models_trained = len(leaderboard)
num_models_ensemble = len(
predictor._trainer.get_minimum_model_set(
predictor._trainer.model_best))
return result(output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
probabilities_labels=prob_labels,
target_is_encoded=False,
models_count=num_models_trained,
models_ensemble_count=num_models_ensemble,
training_duration=training.duration,
predict_duration=predict.duration)
def run(dataset, config):
log.info(f"\n**** mljar-supervised [v{supervised.__version__}] ****\n")
# Mapping of benchmark metrics to MLJAR metrics
metrics_mapping = dict(auc='auc', logloss='logloss', rmse='rmse')
eval_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else "auto"
# Mapping of benchmark task to MLJAR ML task
problem_mapping = dict(
binary="binary_classification",
multiclass="multiclass_classification",
regression="regression",
)
ml_task = problem_mapping.get(
dataset.problem_type
) # if None the AutoML will guess about the ML task
is_classification = config.type == "classification"
results_path = output_subdir("results", config)
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith("_")
}
X_train, y_train = dataset.train.X, dataset.train.y.squeeze()
X_test, y_test = dataset.test.X, dataset.test.y.squeeze()
automl = AutoML(results_path=results_path,
total_time_limit=config.max_runtime_seconds,
random_state=config.seed,
ml_task=ml_task,
eval_metric=eval_metric,
**training_params)
with Timer() as training:
automl.fit(X_train, y_train)
with Timer() as predict:
preds = automl.predict_all(X_test)
predictions, probabilities, probabilities_labels = None, None, None
if is_classification:
# preds is a dataframe with columns ["prediction_LABEL", .., "label"]
if y_train.dtype == bool and preds["label"].dtype == int:
# boolean target produces integer predictions for mljar-supervised <= 0.10.6
# https://github.com/mljar/mljar-supervised/issues/442
preds = preds.rename(
{
"prediction_0": "False",
"prediction_1": "True"
}, axis=1)
preds["label"] = preds["label"].astype(bool)
else:
preds.columns = [
c.replace("prediction_", "", 1) for c in preds.columns
]
predictions = preds["label"].values
probabilities_labels = list(preds.columns)[:-1]
probabilities = preds[probabilities_labels].values
else:
predictions = preds["prediction"].values
# clean the results
if not config.framework_params.get("_save_artifacts", False):
shutil.rmtree(results_path, ignore_errors=True)
return result(output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
probabilities_labels=probabilities_labels,
models_count=len(automl._models),
training_duration=training.duration,
predict_duration=predict.duration)
def run(dataset: Dataset, config: TaskConfig):
log.info(f"\n**** MLNet [v{config.framework_version}] ****\n")
avaible_task_list = ['classification', 'regression']
if config.type not in avaible_task_list:
raise ValueError(f'{config.type} is not supported.')
dir_path = os.path.dirname(os.path.realpath(__file__))
DOTNET_INSTALL_DIR = os.path.join(dir_path, 'lib')
os.environ['DOTNET_ROOT'] = DOTNET_INSTALL_DIR
os.environ['MLNetCLIEnablePredict'] = 'True'
os.environ['MLNET_MAX_THREAD'] = str(config.cores)
mlnet = os.path.join(DOTNET_INSTALL_DIR, 'mlnet')
train_time_in_seconds = config.max_runtime_seconds
sub_command = config.type
# set up MODELBUILDER_AUTOML
MODELBUILDER_AUTOML = config.framework_params.get('automl_type', 'NNI')
os.environ['MODELBUILDER_AUTOML'] = MODELBUILDER_AUTOML
artifacts = config.framework_params.get('_save_artifacts', [])
tmpdir = tempfile.mkdtemp()
tmp_output_folder = os.path.join(tmpdir, str(config.fold))
output_dir = output_subdir(
'models',
config=config) if 'models' in artifacts else tmp_output_folder
log_dir = output_subdir(
'logs', config=config) if 'logs' in artifacts else tmp_output_folder
log_path = os.path.join(log_dir, 'log.txt')
try:
label = dataset.target.index
train_dataset_path = dataset.train.data_path('csv')
test_dataset_path = dataset.test.data_path('csv')
log.info(f'train dataset: {train_dataset_path}')
log.info(f'test dataset: {test_dataset_path}')
cmd = (
f"{mlnet} {sub_command}"
f" --dataset {train_dataset_path} --test-dataset {test_dataset_path} --train-time {train_time_in_seconds}"
f" --label-col {label} --output {os.path.dirname(output_dir)} --name {config.fold}"
f" --verbosity q --log-file-path {log_path}")
with Timer() as training:
run_cmd(cmd)
train_result_json = os.path.join(output_dir,
'{}.mbconfig'.format(config.fold))
if not os.path.exists(train_result_json):
raise NoResultError("MLNet failed producing any prediction.")
with open(train_result_json, 'r') as f:
json_str = f.read()
mb_config = json.loads(json_str)
model_path = os.path.join(output_dir, f"{config.fold}.zip")
output_prediction_path = os.path.join(
log_dir, "prediction.txt"
) # keeping this in log dir as it contains useful error when prediction fails
models_count = len(mb_config['RunHistory']['Trials'])
# predict
predict_cmd = (
f"{mlnet} predict --task-type {config.type}"
f" --model {model_path} --dataset {test_dataset_path} --label-col {dataset.target.name} > {output_prediction_path}"
)
with Timer() as prediction:
run_cmd(predict_cmd)
if config.type == 'classification':
prediction_df = pd.read_csv(output_prediction_path,
dtype={'PredictedLabel': 'object'})
save_predictions(
dataset=dataset,
output_file=config.output_predictions_file,
predictions=prediction_df['PredictedLabel'].values,
truth=dataset.test.y,
probabilities=prediction_df.values[:, :-1],
probabilities_labels=list(
prediction_df.columns.values[:-1]),
)
if config.type == 'regression':
prediction_df = pd.read_csv(output_prediction_path)
save_predictions(
dataset=dataset,
output_file=config.output_predictions_file,
predictions=prediction_df['Score'].values,
truth=dataset.test.y,
)
return dict(
models_count=models_count,
training_duration=training.duration,
predict_duration=prediction.duration,
)
finally:
if 'logs' in artifacts:
logs_zip = os.path.join(log_dir, "logs.zip")
zip_path(log_dir, logs_zip)
clean_dir(log_dir, filter_=lambda p: p != logs_zip)
if 'models' in artifacts:
models_zip = os.path.join(output_dir, "models.zip")
zip_path(output_dir, models_zip)
clean_dir(output_dir, filter_=lambda p: p != models_zip)
shutil.rmtree(tmpdir, ignore_errors=True)
def run(dataset, config):
log.info("\n**** GAMA [v%s] ****", __version__)
log.info("sklearn == %s", sklearn.__version__)
log.info("category_encoders == %s", category_encoders.__version__)
is_classification = (config.type == 'classification')
# Mapping of benchmark metrics to GAMA metrics
metrics_mapping = dict(
acc='accuracy',
auc='roc_auc',
f1='f1',
logloss='neg_log_loss',
mae='neg_mean_absolute_error',
mse='neg_mean_squared_error',
msle='neg_mean_squared_log_error',
r2='r2',
rmse='neg_mean_squared_error',
)
scoring_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else None
if scoring_metric is None:
raise ValueError("Performance metric {} not supported.".format(
config.metric))
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
n_jobs = config.framework_params.get(
'_n_jobs', config.cores
) # useful to disable multicore, regardless of the dataset config
log.info(
'Running GAMA with a maximum time of %ss on %s cores, optimizing %s.',
config.max_runtime_seconds, n_jobs, scoring_metric)
estimator = GamaClassifier if is_classification else GamaRegressor
kwargs = dict(n_jobs=n_jobs,
max_total_time=config.max_runtime_seconds,
scoring=scoring_metric,
random_state=config.seed,
**training_params)
version_leq_20_2_0 = version.parse(__version__) <= version.parse('20.2.0')
if version_leq_20_2_0:
log_file = touch(
os.path.join(output_subdir('logs', config), 'gama.log'))
kwargs['keep_analysis_log'] = log_file
else:
kwargs['max_memory_mb'] = config.max_mem_size_mb
kwargs['output_directory'] = output_subdir('logs', config)
gama_automl = estimator(**kwargs)
X_train, y_train = dataset.train.X, dataset.train.y
# data = file_to_pandas(dataset.train.path, encoding='utf-8')
# X_train, y_train = data.loc[:, data.columns != dataset.target], data.loc[:, dataset.target]
with Timer() as training_timer:
gama_automl.fit(X_train, y_train)
log.info('Predicting on the test set.')
X_test, y_test = dataset.test.X, dataset.test.y
# data = file_to_pandas(dataset.test.path, encoding='utf-8')
# X_test, y_test = data.loc[:, data.columns != dataset.target], data.loc[:, dataset.target]
with Timer() as predict_timer:
predictions = gama_automl.predict(X_test)
if is_classification:
probabilities = gama_automl.predict_proba(X_test)
else:
probabilities = None
return result(output_file=config.output_predictions_file,
predictions=predictions,
probabilities=probabilities,
truth=y_test,
target_is_encoded=False,
models_count=len(gama_automl._final_pop),
training_duration=training_timer.duration,
predict_duration=predict_timer.duration)
def run(dataset, config):
log.info(f"\n**** H2O AutoML [v{h2o.__version__}] ****\n")
save_metadata(config, version=h2o.__version__)
# Mapping of benchmark metrics to H2O metrics
metrics_mapping = dict(acc='mean_per_class_error',
auc='AUC',
logloss='logloss',
mae='mae',
mse='mse',
r2='r2',
rmse='rmse',
rmsle='rmsle')
sort_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else None
if sort_metric is None:
# TODO: Figure out if we are going to blindly pass metrics through, or if we use a strict mapping
log.warning("Performance metric %s not supported, defaulting to AUTO.",
config.metric)
try:
training_params = {
k: v
for k, v in config.framework_params.items()
if not k.startswith('_')
}
nthreads = config.framework_params.get('_nthreads', config.cores)
jvm_memory = str(
round(config.max_mem_size_mb * 2 /
3)) + "M" # leaving 1/3rd of available memory for XGBoost
log.info("Starting H2O cluster with %s cores, %s memory.", nthreads,
jvm_memory)
max_port_range = 49151
min_port_range = 1024
rnd_port = os.getpid() % (max_port_range -
min_port_range) + min_port_range
port = config.framework_params.get('_port', rnd_port)
init_params = config.framework_params.get('_init', {})
if "logs" in config.framework_params.get('_save_artifacts', []):
init_params['ice_root'] = output_subdir("logs", config)
h2o.init(nthreads=nthreads,
port=port,
min_mem_size=jvm_memory,
max_mem_size=jvm_memory,
**init_params)
import_kwargs = {}
# Load train as an H2O Frame, but test as a Pandas DataFrame
log.debug("Loading train data from %s.", dataset.train.path)
train = None
if version.parse(h2o.__version__) >= version.parse(
"3.32.0.3"
): # previous versions may fail to parse correctly some rare arff files using single quotes as enum/string delimiters (pandas also fails on same datasets)
import_kwargs['quotechar'] = '"'
train = h2o.import_file(dataset.train.path,
destination_frame=frame_name(
'train', config),
**import_kwargs)
if not verify_loaded_frame(train, dataset):
h2o.remove(train)
train = None
import_kwargs['quotechar'] = "'"
if not train:
train = h2o.import_file(dataset.train.path,
destination_frame=frame_name(
'train', config),
**import_kwargs)
# train.impute(method='mean')
log.debug("Loading test data from %s.", dataset.test.path)
test = h2o.import_file(dataset.test.path,
destination_frame=frame_name('test', config),
**import_kwargs)
# test.impute(method='mean')
log.info("Running model on task %s, fold %s.", config.name,
config.fold)
log.debug(
"Running H2O AutoML with a maximum time of %ss on %s core(s), optimizing %s.",
config.max_runtime_seconds, config.cores, sort_metric)
aml = H2OAutoML(max_runtime_secs=config.max_runtime_seconds,
sort_metric=sort_metric,
seed=config.seed,
**training_params)
monitor = (
BackendMemoryMonitoring(
frequency_seconds=config.ext.monitoring.frequency_seconds,
check_on_exit=True,
verbosity=config.ext.monitoring.verbosity)
if config.framework_params.get('_monitor_backend', False)
# else contextlib.nullcontext # Py 3.7+ only
else contextlib.contextmanager(iter)([0]))
with utils.Timer() as training:
with monitor:
aml.train(y=dataset.target.index, training_frame=train)
if not aml.leader:
raise FrameworkError(
"H2O could not produce any model in the requested time.")
with utils.Timer() as predict:
preds = aml.predict(test)
preds = extract_preds(preds, test, dataset=dataset)
save_artifacts(aml, dataset=dataset, config=config)
return result(output_file=config.output_predictions_file,
predictions=preds.predictions,
truth=preds.truth,
probabilities=preds.probabilities,
probabilities_labels=preds.probabilities_labels,
models_count=len(aml.leaderboard),
training_duration=training.duration,
predict_duration=predict.duration)
finally:
if h2o.connection():
# h2o.remove_all()
h2o.connection().close()
if h2o.connection().local_server:
h2o.connection().local_server.shutdown()
def run(dataset, config):
log.info(f"\n**** mljar-supervised [v{supervised.__version__}] ****\n")
save_metadata(config, version=supervised.__version__)
# Mapping of benchmark metrics to MLJAR metrics
metrics_mapping = dict(auc='auc', logloss='logloss', rmse='rmse')
eval_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else "auto"
# Mapping of benchmark task to MLJAR ML task
problem_mapping = dict(
binary="binary_classification",
multiclass="multiclass_classification",
regression="regression",
)
ml_task = problem_mapping.get(
dataset.problem_type
) # if None the AutoML will guess about the ML task
is_classification = config.type == "classification"
results_path = output_subdir("results", config)
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith("_")
}
column_names, _ = zip(*dataset.columns)
column_types = dict(dataset.columns)
label = dataset.target.name
train = pd.DataFrame(dataset.train.data,
columns=column_names).astype(column_types, copy=False)
X_train = train.drop(columns=label)
y_train = train[label]
test = pd.DataFrame(dataset.test.data,
columns=column_names).astype(column_types, copy=False)
X_test = test.drop(columns=label)
y_test = test[label]
automl = AutoML(results_path=results_path,
total_time_limit=config.max_runtime_seconds,
random_state=config.seed,
ml_task=ml_task,
eval_metric=eval_metric,
**training_params)
with utils.Timer() as training:
automl.fit(X_train, y_train)
with utils.Timer() as predict:
preds = automl.predict_all(X_test)
predictions, probabilities = None, None
if is_classification:
predictions = preds["label"].values
cols = [f"prediction_{c}" for c in np.unique(y_train)]
probabilities = preds[cols].values
else:
predictions = preds["prediction"].values
# clean the results
if not config.framework_params.get("_save_artifacts", False):
shutil.rmtree(results_path, ignore_errors=True)
return result(output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
models_count=len(automl._models),
training_duration=training.duration,
predict_duration=predict.duration)
def save_artifacts(automl, dataset, config):
artifacts = config.framework_params.get('_save_artifacts', ['leaderboard'])
try:
models_artifacts = []
lb = automl.leaderboard.as_data_frame()
log.debug("Leaderboard:\n%s", lb.to_string())
if 'leaderboard' in artifacts:
models_dir = output_subdir("models", config)
lb_path = os.path.join(models_dir, "leaderboard.csv")
write_csv(lb, lb_path)
models_artifacts.append(lb_path)
models_pat = re.compile(r"models(\[(json|binary|mojo)(?:,(\d+))?\])?")
models = list(filter(models_pat.fullmatch, artifacts))
for m in models:
models_dir = output_subdir("models", config)
all_models_se = next(
(mid for mid in lb['model_id']
if mid.startswith("StackedEnsemble_AllModels")), None)
match = models_pat.fullmatch(m)
mformat = match.group(2) or 'json'
topN = int(match.group(3) or -1)
if topN < 0 and mformat != 'json' and all_models_se:
models_artifacts.append(
save_model(all_models_se,
dest_dir=models_dir,
mformat=mformat))
else:
count = 0
for mid in lb['model_id']:
if topN < 0 or count < topN:
save_model(mid, dest_dir=models_dir, mformat=mformat)
count += 1
else:
break
models_archive = os.path.join(models_dir,
f"models_{mformat}.zip")
utils.zip_path(models_dir,
models_archive,
filtr=lambda p: p not in models_artifacts)
models_artifacts.append(models_archive)
def delete(path, isdir):
if not isdir and path not in models_artifacts and os.path.splitext(
path)[1] in ['.json', '.zip', '']:
os.remove(path)
utils.walk_apply(models_dir, delete, max_depth=0)
if 'model_predictions' in artifacts:
predictions_dir = output_subdir("predictions", config)
test = h2o.get_frame(frame_name('test', config))
for mid in lb['model_id']:
model = h2o.get_model(mid)
h2o_preds = model.predict(test)
preds = extract_preds(h2o_preds, test, dataset=dataset)
if preds.probabilities_labels is None:
preds.probabilities_labels = preds.h2o_labels
write_preds(
preds, os.path.join(predictions_dir, mid,
'predictions.csv'))
utils.zip_path(
predictions_dir,
os.path.join(predictions_dir, "model_predictions.zip"))
def delete(path, isdir):
if isdir:
shutil.rmtree(path, ignore_errors=True)
utils.walk_apply(predictions_dir, delete, max_depth=0)
if 'logs' in artifacts:
logs_dir = output_subdir("logs", config)
logs_zip = os.path.join(logs_dir, "h2o_logs.zip")
utils.zip_path(logs_dir, logs_zip)
# h2o.download_all_logs(dirname=logs_dir)
def delete(path, isdir):
if isdir:
shutil.rmtree(path, ignore_errors=True)
elif path != logs_zip:
os.remove(path)
utils.walk_apply(logs_dir, delete, max_depth=0)
except Exception:
log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(automl, dataset, config):
artifacts = config.framework_params.get('_save_artifacts', ['leaderboard'])
try:
models_artifacts = []
lb_pat = re.compile(r"leaderboard(?:\[(.*)\])?")
lb_match = next((lb_pat.fullmatch(a) for a in artifacts), None)
if lb_match:
lb_ext = list(
filter(None, re.split("[,; ]", (lb_match.group(1) or ""))))
lb = h2o.automl.get_leaderboard(automl, lb_ext).as_data_frame()
models_dir = output_subdir("models", config)
lb_path = os.path.join(models_dir, "leaderboard.csv")
write_csv(lb, lb_path)
models_artifacts.append(lb_path)
else:
lb = automl.leaderboard.as_data_frame()
log.debug("Leaderboard:\n%s", lb.to_string())
models_pat = re.compile(r"models(\[(json|binary|mojo)(?:,(\d+))?\])?")
models = list(filter(models_pat.fullmatch, artifacts))
for m in models:
models_dir = output_subdir("models", config)
all_models_se = next(
(mid for mid in lb['model_id']
if mid.startswith("StackedEnsemble_AllModels")), None)
match = models_pat.fullmatch(m)
mformat = match.group(2) or 'json'
topN = int(match.group(3) or -1)
if topN < 0 and mformat != 'json' and all_models_se:
models_artifacts.append(
save_model(all_models_se,
dest_dir=models_dir,
mformat=mformat))
else:
count = 0
for mid in lb['model_id']:
if topN < 0 or count < topN:
save_model(mid, dest_dir=models_dir, mformat=mformat)
count += 1
else:
break
models_archive = os.path.join(models_dir,
f"models_{mformat}.zip")
zip_path(models_dir,
models_archive,
filter_=lambda p: p not in models_artifacts)
models_artifacts.append(models_archive)
clean_dir(models_dir,
filter_=lambda p: p not in models_artifacts and os.
path.splitext(p)[1] in ['.json', '.zip', ''])
if 'model_predictions' in artifacts:
predictions_dir = output_subdir("predictions", config)
test = h2o.get_frame(frame_name('test', config))
for mid in lb['model_id']:
model = h2o.get_model(mid)
h2o_preds = model.predict(test)
preds = extract_preds(h2o_preds, test, dataset=dataset)
if preds.probabilities_labels is None:
preds.probabilities_labels = preds.h2o_labels
write_preds(
preds, os.path.join(predictions_dir, mid,
'predictions.csv'))
predictions_zip = os.path.join(predictions_dir,
"model_predictions.zip")
zip_path(predictions_dir, predictions_zip)
clean_dir(predictions_dir, filter_=lambda p: os.path.isdir(p))
if 'logs' in artifacts:
logs_dir = output_subdir("logs", config)
logs_zip = os.path.join(logs_dir, "h2o_logs.zip")
zip_path(logs_dir, logs_zip)
# h2o.download_all_logs(dirname=logs_dir)
clean_dir(logs_dir, filter_=lambda p: p != logs_zip)
write_csv(automl.event_log.as_data_frame(),
os.path.join(logs_dir, 'aml_event_log.csv'))
except Exception:
log.debug("Error when saving artifacts.", exc_info=True)
def run(dataset, config):
log.info("\n**** genens ****\n")
is_classification = config.type == 'classification'
if not is_classification:
Warning("Regression not supported.")
return None
# Mapping of benchmark metrics to TPOT metrics
metrics_mapping = {
'acc': get_scorer('accuracy'),
'auc': get_scorer('roc_auc'),
'f1': get_scorer('f1'),
'logloss': get_scorer('neg_log_loss'),
'mae': get_scorer('neg_mean_absolute_error'),
'mse': get_scorer('neg_mean_squared_error'),
'msle': get_scorer('neg_mean_squared_log_error'),
'r2': get_scorer('r2')
}
scoring_metric = metrics_mapping[
config.metric] if config.metric in metrics_mapping else None
if scoring_metric is None:
raise ValueError("Performance metric {} not supported.".format(
config.metric))
X_train, X_test = dataset.train.X_enc, dataset.test.X_enc
y_train, y_test = dataset.train.y_enc, dataset.test.y_enc
training_params = {
k: v
for k, v in config.framework_params.items() if not k.startswith('_')
}
n_jobs = config.framework_params.get(
'_n_jobs', config.cores
) # useful to disable multicore, regardless of the dataset config
sample_size = config.framework_params.get('_sample_size', None)
if sample_size is not None:
evaluator = SampleCrossValEvaluator(sample_size=sample_size,
per_gen=True,
cv_k=5)
else:
evaluator = CrossValEvaluator(cv_k=5)
print(f"Chosen sample size: {sample_size}.")
print(f'cv_k: {evaluator.cv_k}')
training_params['evaluator'] = evaluator
runtime_s = config.max_runtime_seconds
if runtime_s >= 600:
runtime_s -= 5 * 60 # avoid premature process termination
elif runtime_s > 10:
runtime_s -= 5
if not config.framework_params.get('disable_logging', True):
log_path = os.path.join(output_subdir('logs', config),
'evo_log_file.txt')
else:
log_path = None
print(f"Setting time limit to {runtime_s} seconds.")
log.info(
'Running genens with a maximum time of %ss on %s cores, optimizing %s.',
runtime_s, n_jobs, scoring_metric)
if config.seed is not None:
# random state is yet to be unified in genens
np.random.seed(config.seed)
random.seed(config.seed)
print(f'Training params: {training_params}')
estimator = GenensClassifier if is_classification else GenensRegressor
genens_est = estimator(n_jobs=n_jobs,
max_evo_seconds=runtime_s,
scorer=scoring_metric,
log_path=log_path,
**training_params)
with utils.Timer() as training:
genens_est.fit(X_train, y_train)
log.info('Predicting on the test set.')
best_pipe = genens_est.get_best_pipelines()[0]
best_pipe.fit(X_train, y_train)
predictions = best_pipe.predict(X_test)
try:
probabilities = best_pipe.predict_proba(
X_test) if is_classification else None
except AttributeError:
target_values_enc = dataset.target.label_encoder.transform(
dataset.target.values)
probabilities = utils.Encoder(
'one-hot', target=False,
encoded_type=float).fit(target_values_enc).transform(predictions)
save_artifacts(genens_est, config)
return result(output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
target_is_encoded=is_classification,
models_count=len(genens_est.get_best_pipelines()),
training_duration=training.duration)
def run(dataset, config):
log.info(f"\n**** AutoGluon [v{__version__}] ****\n")
save_metadata(config, version=__version__)
metrics_mapping = dict(
acc=metrics.accuracy,
auc=metrics.roc_auc,
f1=metrics.f1,
logloss=metrics.log_loss,
mae=metrics.mean_absolute_error,
mse=metrics.mean_squared_error,
r2=metrics.r2,
rmse=metrics.root_mean_squared_error,
)
label = dataset.target.name
problem_type = dataset.problem_type
perf_metric = metrics_mapping[config.metric] if config.metric in metrics_mapping else None
if perf_metric is None:
# TODO: figure out if we are going to blindly pass metrics through, or if we use a strict mapping
log.warning("Performance metric %s not supported.", config.metric)
is_classification = config.type == 'classification'
training_params = {k: v for k, v in config.framework_params.items() if not k.startswith('_')}
load_raw = config.framework_params.get('_load_raw', False)
if load_raw:
train, test = load_data_raw(dataset=dataset)
else:
column_names, _ = zip(*dataset.columns)
column_types = dict(dataset.columns)
train = pd.DataFrame(dataset.train.data, columns=column_names).astype(column_types, copy=False)
print(f"Columns dtypes:\n{train.dtypes}")
test = pd.DataFrame(dataset.test.data, columns=column_names).astype(column_types, copy=False)
del dataset
gc.collect()
output_dir = output_subdir("models", config)
with utils.Timer() as training:
predictor = TabularPredictor(
label=label,
eval_metric=perf_metric.name,
path=output_dir,
problem_type=problem_type,
).fit(
train_data=train,
time_limit=config.max_runtime_seconds,
**training_params
)
del train
y_test = test[label]
test = test.drop(columns=label)
if is_classification:
with utils.Timer() as predict:
probabilities = predictor.predict_proba(test, as_multiclass=True)
predictions = probabilities.idxmax(axis=1).to_numpy()
else:
with utils.Timer() as predict:
predictions = predictor.predict(test, as_pandas=False)
probabilities = None
prob_labels = probabilities.columns.values.tolist() if probabilities is not None else None
leaderboard = predictor.leaderboard(silent=True) # Removed test data input to avoid long running computation, remove 7200s timeout limitation to re-enable
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 1000):
print(leaderboard)
save_artifacts(predictor, leaderboard, config)
num_models_trained = len(leaderboard)
if predictor._trainer.model_best is not None:
num_models_ensemble = len(predictor._trainer.get_minimum_model_set(predictor._trainer.model_best))
else:
num_models_ensemble = 1
return result(output_file=config.output_predictions_file,
predictions=predictions,
truth=y_test,
probabilities=probabilities,
probabilities_labels=prob_labels,
target_is_encoded=False,
models_count=num_models_trained,
models_ensemble_count=num_models_ensemble,
training_duration=training.duration,
predict_duration=predict.duration)
