def save_artifacts(predictor, leaderboard, config): artifacts = config.framework_params.get('_save_artifacts', ['leaderboard']) try: models_dir = output_subdir("models", config) shutil.rmtree(os.path.join(models_dir, "utils"), ignore_errors=True) if 'leaderboard' in artifacts: save_pd.save(path=os.path.join(models_dir, "leaderboard.csv"), df=leaderboard) if 'info' in artifacts: ag_info = predictor.info() info_dir = output_subdir("info", config) save_pkl.save(path=os.path.join(info_dir, "info.pkl"), object=ag_info) if 'models' in artifacts: utils.zip_path(models_dir, os.path.join(models_dir, "models.zip")) def delete(path, isdir): if isdir: shutil.rmtree(path, ignore_errors=True) elif os.path.splitext(path)[1] == '.pkl': os.remove(path) utils.walk_apply(models_dir, delete, max_depth=0) except Exception: log.warning("Error when saving artifacts.", exc_info=True)
def save_artifacts(estimator, config, start, dataset): try: models_repr = estimator.show_models() log.debug("Trained Ensemble:\n%s", models_repr) artifacts = config.framework_params.get('_save_artifacts', []) if 'models' in artifacts: config_file = os.path.join(output_subdir('models', config), 'configspace.pkl') try: with open(config_file, 'wb') as f: cs = estimator.get_configuration_space( dataset.train.X_enc, dataset.train.y_enc) print(cs) pickle.dump(cs, f) except Exception as ex: log.exception('Error: {}'.format(ex)) start_file = os.path.join(output_subdir('models', config), 'start.txt') with open(start_file, 'w') as f: f.write(str(start)) models_file = os.path.join(output_subdir('models', config), 'models.txt') with open(models_file, 'w') as f: f.write(models_repr) for idx, automl in enumerate(estimator._automl): runhistory_file = os.path.join( output_subdir('models', config), 'runhistory_{}.json'.format(idx)) automl.runhistory_.save_json(runhistory_file) except Exception: log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(automl, dataset, config): artifacts = config.framework_params.get('_save_artifacts', ['leaderboard']) try: lb = automl.leaderboard.as_data_frame() log.debug("Leaderboard:\n%s", lb.to_string()) if 'leaderboard' in artifacts: models_dir = output_subdir("models", config) write_csv(lb, os.path.join(models_dir, "leaderboard.csv")) if 'models' in artifacts: models_dir = output_subdir("models", config) all_models_se = next( (mid for mid in lb['model_id'] if mid.startswith("StackedEnsemble_AllModels")), None) mformat = 'mojo' if 'mojos' in artifacts else 'json' if all_models_se and mformat == 'mojo': save_model(all_models_se, dest_dir=models_dir, mformat=mformat) else: for mid in lb['model_id']: save_model(mid, dest_dir=models_dir, mformat=mformat) models_archive = os.path.join(models_dir, "models.zip") zip_path(models_dir, models_archive) def delete(path, isdir): if path != models_archive and os.path.splitext( path)[1] in ['.json', '.zip']: os.remove(path) walk_apply(models_dir, delete, max_depth=0) if 'models_predictions' in artifacts: predictions_dir = output_subdir("predictions", config) test = h2o.get_frame(frame_name('test', config)) for mid in lb['model_id']: model = h2o.get_model(mid) save_predictions(model, test, dataset=dataset, config=config, predictions_file=os.path.join( predictions_dir, mid, 'predictions.csv'), preview=False) zip_path(predictions_dir, os.path.join(predictions_dir, "models_predictions.zip")) def delete(path, isdir): if isdir: shutil.rmtree(path, ignore_errors=True) walk_apply(predictions_dir, delete, max_depth=0) if 'logs' in artifacts: logs_dir = output_subdir("logs", config) h2o.download_all_logs(dirname=logs_dir) except Exception: log.debug("Error when saving artifacts.", exc_info=True)
def run(dataset, config): log.info("\n**** mljar-supervised ****\n") column_names, _ = zip(*dataset.columns) column_types = dict(dataset.columns) X_train = pd.DataFrame(dataset.train.X, columns=column_names).astype(column_types, copy=False) X_test = pd.DataFrame(dataset.test.X, columns=column_names).astype(column_types, copy=False) y_train = dataset.train.y.flatten() y_test = dataset.test.y.flatten() problem_mapping = dict( binary="binary_classification", multiclass="multiclass_classification", regression="regression", ) is_classification = config.type == "classification" ml_task = problem_mapping.get( dataset.problem_type ) # if None the AutoML will guess about the ML task results_path = output_subdir("results", config) training_params = { k: v for k, v in config.framework_params.items() if not k.startswith("_") } automl = AutoML(results_path=results_path, total_time_limit=config.max_runtime_seconds, seed=config.seed, ml_task=ml_task, **training_params) with Timer() as training: automl.fit(X_train, y_train) preds = automl.predict(X_test) predictions, probabilities = None, None if is_classification: predictions = preds["label"].values probabilities = preds[preds.columns[:-1]].values else: predictions = preds["prediction"].values # clean the results if not config.framework_params.get("_save_artifacts", False): shutil.rmtree(results_path, ignore_errors=True) return result( output_file=config.output_predictions_file, predictions=predictions, truth=y_test, probabilities=probabilities, models_count=len(automl._models), training_duration=training.duration, )
def run(dataset, config): log.info(f"\n**** FLAML [v{__version__}] ****\n") X_train, y_train = dataset.train.X, dataset.train.y.squeeze() X_test, y_test = dataset.test.X, dataset.test.y.squeeze() is_classification = config.type == 'classification' time_budget = config.max_runtime_seconds n_jobs = config.framework_params.get('_n_jobs', config.cores) log.info("Running FLAML with {} number of cores".format(config.cores)) aml = AutoML() # Mapping of benchmark metrics to flaml metrics metrics_mapping = dict( acc='accuracy', auc='roc_auc', f1='f1', logloss='log_loss', mae='mae', mse='mse', rmse='rmse', r2='r2', ) perf_metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else 'auto' if perf_metric is None: log.warning("Performance metric %s not supported.", config.metric) training_params = { k: v for k, v in config.framework_params.items() if not k.startswith('_') } log_dir = output_subdir("logs", config) flaml_log_file_name = os.path.join(log_dir, "flaml.log") with Timer() as training: aml.fit(X_train, y_train, metric=perf_metric, task=config.type, n_jobs=n_jobs, log_file_name=flaml_log_file_name, time_budget=time_budget, **training_params) with Timer() as predict: predictions = aml.predict(X_test) probabilities = aml.predict_proba(X_test) if is_classification else None labels = aml.classes_ if is_classification else None return result( output_file=config.output_predictions_file, probabilities=probabilities, predictions=predictions, truth=y_test, models_count=len(aml.config_history), training_duration=training.duration, predict_duration=predict.duration, probabilities_labels=labels, )
def save_artifacts(estimator: Union[GenensClassifier, GenensRegressor], config): try: artifacts = config.framework_params.get('_save_artifacts', False) if 'pickle_models' in artifacts: models_dir = os.path.join(output_subdir('pickle_models', config)) # pickle top 3 best pipelines for i, pipe in enumerate(estimator.get_best_pipelines()): with open(models_dir + '/pipeline{}.pickle'.format(i), 'wb') as pickle_file: pickle.dump(pipe, pickle_file, pickle.HIGHEST_PROTOCOL) if 'models' in artifacts: models_dir = os.path.join(output_subdir('models', config)) # top 3 individual fitness values with open(models_dir + '/ind-fitness.txt', 'w+') as out_file: best_inds = estimator.get_best_pipelines(as_individuals=True) for i, ind in enumerate(best_inds): out_file.write('Individual {}: Score {}\n'.format( i, ind.fitness.values)) # individual tree if 'models_png' in artifacts: from genens.render.graph import create_graph models_dir = os.path.join(output_subdir('models', config)) for i, ind in enumerate(best_inds): create_graph(ind, models_dir + '/graph{}.png'.format(i)) if 'log' in artifacts: log_dir = os.path.join(output_subdir('logs', config)) # write logbook string representation to output dir with open(log_dir + '/logbook.txt', 'w+') as log_file: log_file.write(estimator.logbook.__str__() + '\n') # evolution plot export_plot(estimator, log_dir + '/result.png') except: log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(chain, config): try: artifacts = config.framework_params.get('_save_artifacts', False) if 'models' in artifacts: models_file = os.path.join(output_subdir('models', config), 'model.json') chain.save_chain(models_file) except Exception: log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(predictor, leaderboard, config): artifacts = config.framework_params.get('_save_artifacts', ['leaderboard']) try: if 'leaderboard' in artifacts: leaderboard_dir = output_subdir("leaderboard", config) save_pd.save(path=os.path.join(leaderboard_dir, "leaderboard.csv"), df=leaderboard) if 'info' in artifacts: ag_info = predictor.info() info_dir = output_subdir("info", config) save_pkl.save(path=os.path.join(info_dir, "info.pkl"), object=ag_info) if 'models' in artifacts: shutil.rmtree(os.path.join(predictor.path, "utils"), ignore_errors=True) models_dir = output_subdir("models", config) utils.zip_path(predictor.path, os.path.join(models_dir, "models.zip")) except Exception: log.warning("Error when saving artifacts.", exc_info=True)
def save_artifacts(estimator, config): try: log.debug("All individuals :\n%s", list(estimator.evaluated_individuals_.items())) models = estimator.pareto_front_fitted_pipelines_ hall_of_fame = list(zip(reversed(estimator._pareto_front.keys), estimator._pareto_front.items)) artifacts = config.framework_params.get('_save_artifacts', False) if 'models' in artifacts: models_file = os.path.join(output_subdir('models', config), 'models.txt') with open(models_file, 'w') as f: for m in hall_of_fame: pprint.pprint(dict( fitness=str(m[0]), model=str(m[1]), pipeline=models[str(m[1])], ), stream=f) run_history_file = os.path.join(output_subdir('models', config), 'runhistory.pkl') with open(run_history_file, 'wb') as f: pickle.dump(estimator.run_history, f) except Exception: log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(automl, config): try: artifacts = config.framework_params.get('_save_artifacts', []) models_dir = output_subdir("models", config) if 'models' in artifacts: with open(os.path.join(models_dir, 'automl.pickle'), 'wb') as f: pickle.dump(automl, f) except Exception: log.warning("Error when saving artifacts.", exc_info=True)
def save_artifacts(estimator, config): try: models_repr = estimator.show_models() log.debug("Trained Ensemble:\n%s", models_repr) artifacts = config.framework_params.get('_save_artifacts', []) if 'models' in artifacts: models_file = os.path.join(output_subdir('models', config), 'models.txt') with open(models_file, 'w') as f: f.write(models_repr) except Exception: log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(estimator, config): try: models_repr = estimator.show_models() log.debug("Trained Ensemble:\n%s", models_repr) artifacts = config.framework_params.get('_save_artifacts', []) if 'models' in artifacts: models_file = os.path.join(output_subdir('models', config), 'models.txt') with open(models_file, 'w') as f: f.write(models_repr) if 'debug_as_files' in artifacts or 'debug_as_zip' in artifacts: print('Saving debug artifacts!') debug_dir = output_subdir('debug', config) ignore_extensions = [ '.npy', '.pcs', '.model', '.cv_model', '.ensemble', '.pkl' ] tmp_directory = estimator.automl_._backend.temporary_directory if 'debug_as_files' in artifacts: def _copy(filename, **_): dst = filename.replace(tmp_directory, debug_dir + '/') os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copyfile(filename, dst) walk_apply( tmp_directory, _copy, filter_=lambda path: (os.path.splitext(path)[1] not in ignore_extensions and not os.path.isdir(path)), ) else: zip_path(tmp_directory, os.path.join(debug_dir, "artifacts.zip"), filter_=lambda p: os.path.splitext(p)[1] not in ignore_extensions) except Exception as e: log.debug("Error when saving artifacts= {e}.".format(e), exc_info=True)
def run(dataset, config): jar_file = glob.glob("{here}/lib/mlplan/mlplan-cli*.jar".format( here=os.path.dirname(__file__)))[0] version = re.match(r".*/mlplan-cli-(.*).jar", jar_file)[1] log.info(f"\n**** ML-Plan [v{version}] ****\n") is_classification = config.type == 'classification' # Mapping of benchmark metrics to Weka metrics metrics_mapping = dict(acc='ERRORRATE', auc='AUC', logloss='LOGLOSS', f1='F1', r2='R2', rmse='ROOT_MEAN_SQUARED_ERROR', mse='MEAN_SQUARED_ERROR', rmsle='ROOT_MEAN_SQUARED_LOGARITHM_ERROR', mae='MEAN_ABSOLUTE_ERROR') metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else None if metric is None: raise ValueError('Performance metric {} is not supported.'.format( config.metric)) train_file = dataset.train.path test_file = dataset.test.path training_params = { k: v for k, v in config.framework_params.items() if not k.startswith('_') } backend = config.framework_params.get('_backend', 'weka') if backend == "weka": mem_limit = str(max(config.max_mem_size_mb - 1024, 2048)) else: mem_limit = str( max(round((config.max_mem_size_mb - 1024) / config.cores), 2048)) mode = backend if config.type == 'regression': mode += '-regression' log.info( "Running ML-Plan with backend %s in mode %s and a maximum time of %ss on %s cores with %sMB for the JVM, optimizing %s.", backend, mode, config.max_runtime_seconds, config.cores, config.max_mem_size_mb, metric) log.info("Environment: %s", os.environ) predictions_file = os.path.join(output_subdir('mlplan_out', config), 'predictions.csv') statistics_file = os.path.join(output_subdir('mlplan_out', config), 'statistics.json') #tmp_dir = output_subdir('mlplan_tmp', config) cmd_root = f"java -jar -Xmx{mem_limit}M {jar_file}" with tempfile.TemporaryDirectory() as tmp_dir: cmd_params = dict( f='"{}"'.format(train_file), p='"{}"'.format(test_file), t=config.max_runtime_seconds, ncpus=config.cores, l=metric, m=mode, s=config.seed, # weka accepts only int16 as seeds ooab=predictions_file, os=statistics_file, tmp=tmp_dir, **training_params) cmd = cmd_root + ''.join( [" -{} {}".format(k, v) for k, v in cmd_params.items()]) with utils.Timer() as training: utils.run_cmd(cmd, _live_output_=True) with open(statistics_file, 'r') as f: stats = json.load(f) predictions = stats["predictions"] truth = stats["truth"] numEvals = stats["num_evaluations"] # only for classification tasks we have probabilities available, thus check whether the json contains the respective fields if "probabilities" in stats and "probabilities_labels" in stats: probabilities = stats["probabilities"] probabilities_labels = stats["probabilities_labels"] else: probabilities = [] probabilities_labels = [] return result(output_file=config.output_predictions_file, predictions=predictions, truth=truth, probabilities=probabilities, probabilities_labels=probabilities_labels, target_is_encoded=is_classification, models_count=numEvals, training_duration=training.duration)
def run(dataset, config): log.info(f"\n**** AutoGluon [v{__version__}] ****\n") save_metadata(config, version=__version__) metrics_mapping = dict( acc=metrics.accuracy, auc=metrics.roc_auc, f1=metrics.f1, logloss=metrics.log_loss, mae=metrics.mean_absolute_error, mse=metrics.mean_squared_error, r2=metrics.r2, # rmse=metrics.root_mean_squared_error, # metrics.root_mean_squared_error incorrectly registered in autogluon REGRESSION_METRICS rmse=metrics. mean_squared_error, # for now, we can let autogluon optimize training on mse: anyway we compute final score from predictions. ) perf_metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else None if perf_metric is None: # TODO: figure out if we are going to blindly pass metrics through, or if we use a strict mapping log.warning("Performance metric %s not supported.", config.metric) is_classification = config.type == 'classification' training_params = { k: v for k, v in config.framework_params.items() if not k.startswith('_') } column_names, _ = zip(*dataset.columns) column_types = dict(dataset.columns) train = pd.DataFrame(dataset.train.data, columns=column_names).astype(column_types, copy=False) label = dataset.target.name print(f"Columns dtypes:\n{train.dtypes}") output_dir = output_subdir("models", config) with utils.Timer() as training: predictor = task.fit(train_data=train, label=label, problem_type=dataset.problem_type, output_directory=output_dir, time_limits=config.max_runtime_seconds, eval_metric=perf_metric.name, **training_params) test = pd.DataFrame(dataset.test.data, columns=column_names).astype(column_types, copy=False) X_test = test.drop(columns=label) y_test = test[label] with utils.Timer() as predict: predictions = predictor.predict(X_test) probabilities = predictor.predict_proba( dataset=X_test, as_pandas=True, as_multiclass=True) if is_classification else None prob_labels = probabilities.columns.values.tolist( ) if probabilities is not None else None leaderboard = predictor._learner.leaderboard(X_test, y_test, silent=True) with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 1000): print(leaderboard) save_artifacts(predictor, leaderboard, config) num_models_trained = len(leaderboard) num_models_ensemble = len( predictor._trainer.get_minimum_model_set( predictor._trainer.model_best)) return result(output_file=config.output_predictions_file, predictions=predictions, truth=y_test, probabilities=probabilities, probabilities_labels=prob_labels, target_is_encoded=False, models_count=num_models_trained, models_ensemble_count=num_models_ensemble, training_duration=training.duration, predict_duration=predict.duration)
def run(dataset, config): log.info(f"\n**** mljar-supervised [v{supervised.__version__}] ****\n") # Mapping of benchmark metrics to MLJAR metrics metrics_mapping = dict(auc='auc', logloss='logloss', rmse='rmse') eval_metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else "auto" # Mapping of benchmark task to MLJAR ML task problem_mapping = dict( binary="binary_classification", multiclass="multiclass_classification", regression="regression", ) ml_task = problem_mapping.get( dataset.problem_type ) # if None the AutoML will guess about the ML task is_classification = config.type == "classification" results_path = output_subdir("results", config) training_params = { k: v for k, v in config.framework_params.items() if not k.startswith("_") } X_train, y_train = dataset.train.X, dataset.train.y.squeeze() X_test, y_test = dataset.test.X, dataset.test.y.squeeze() automl = AutoML(results_path=results_path, total_time_limit=config.max_runtime_seconds, random_state=config.seed, ml_task=ml_task, eval_metric=eval_metric, **training_params) with Timer() as training: automl.fit(X_train, y_train) with Timer() as predict: preds = automl.predict_all(X_test) predictions, probabilities, probabilities_labels = None, None, None if is_classification: # preds is a dataframe with columns ["prediction_LABEL", .., "label"] if y_train.dtype == bool and preds["label"].dtype == int: # boolean target produces integer predictions for mljar-supervised <= 0.10.6 # https://github.com/mljar/mljar-supervised/issues/442 preds = preds.rename( { "prediction_0": "False", "prediction_1": "True" }, axis=1) preds["label"] = preds["label"].astype(bool) else: preds.columns = [ c.replace("prediction_", "", 1) for c in preds.columns ] predictions = preds["label"].values probabilities_labels = list(preds.columns)[:-1] probabilities = preds[probabilities_labels].values else: predictions = preds["prediction"].values # clean the results if not config.framework_params.get("_save_artifacts", False): shutil.rmtree(results_path, ignore_errors=True) return result(output_file=config.output_predictions_file, predictions=predictions, truth=y_test, probabilities=probabilities, probabilities_labels=probabilities_labels, models_count=len(automl._models), training_duration=training.duration, predict_duration=predict.duration)
def run(dataset: Dataset, config: TaskConfig): log.info(f"\n**** MLNet [v{config.framework_version}] ****\n") avaible_task_list = ['classification', 'regression'] if config.type not in avaible_task_list: raise ValueError(f'{config.type} is not supported.') dir_path = os.path.dirname(os.path.realpath(__file__)) DOTNET_INSTALL_DIR = os.path.join(dir_path, 'lib') os.environ['DOTNET_ROOT'] = DOTNET_INSTALL_DIR os.environ['MLNetCLIEnablePredict'] = 'True' os.environ['MLNET_MAX_THREAD'] = str(config.cores) mlnet = os.path.join(DOTNET_INSTALL_DIR, 'mlnet') train_time_in_seconds = config.max_runtime_seconds sub_command = config.type # set up MODELBUILDER_AUTOML MODELBUILDER_AUTOML = config.framework_params.get('automl_type', 'NNI') os.environ['MODELBUILDER_AUTOML'] = MODELBUILDER_AUTOML artifacts = config.framework_params.get('_save_artifacts', []) tmpdir = tempfile.mkdtemp() tmp_output_folder = os.path.join(tmpdir, str(config.fold)) output_dir = output_subdir( 'models', config=config) if 'models' in artifacts else tmp_output_folder log_dir = output_subdir( 'logs', config=config) if 'logs' in artifacts else tmp_output_folder log_path = os.path.join(log_dir, 'log.txt') try: label = dataset.target.index train_dataset_path = dataset.train.data_path('csv') test_dataset_path = dataset.test.data_path('csv') log.info(f'train dataset: {train_dataset_path}') log.info(f'test dataset: {test_dataset_path}') cmd = ( f"{mlnet} {sub_command}" f" --dataset {train_dataset_path} --test-dataset {test_dataset_path} --train-time {train_time_in_seconds}" f" --label-col {label} --output {os.path.dirname(output_dir)} --name {config.fold}" f" --verbosity q --log-file-path {log_path}") with Timer() as training: run_cmd(cmd) train_result_json = os.path.join(output_dir, '{}.mbconfig'.format(config.fold)) if not os.path.exists(train_result_json): raise NoResultError("MLNet failed producing any prediction.") with open(train_result_json, 'r') as f: json_str = f.read() mb_config = json.loads(json_str) model_path = os.path.join(output_dir, f"{config.fold}.zip") output_prediction_path = os.path.join( log_dir, "prediction.txt" ) # keeping this in log dir as it contains useful error when prediction fails models_count = len(mb_config['RunHistory']['Trials']) # predict predict_cmd = ( f"{mlnet} predict --task-type {config.type}" f" --model {model_path} --dataset {test_dataset_path} --label-col {dataset.target.name} > {output_prediction_path}" ) with Timer() as prediction: run_cmd(predict_cmd) if config.type == 'classification': prediction_df = pd.read_csv(output_prediction_path, dtype={'PredictedLabel': 'object'}) save_predictions( dataset=dataset, output_file=config.output_predictions_file, predictions=prediction_df['PredictedLabel'].values, truth=dataset.test.y, probabilities=prediction_df.values[:, :-1], probabilities_labels=list( prediction_df.columns.values[:-1]), ) if config.type == 'regression': prediction_df = pd.read_csv(output_prediction_path) save_predictions( dataset=dataset, output_file=config.output_predictions_file, predictions=prediction_df['Score'].values, truth=dataset.test.y, ) return dict( models_count=models_count, training_duration=training.duration, predict_duration=prediction.duration, ) finally: if 'logs' in artifacts: logs_zip = os.path.join(log_dir, "logs.zip") zip_path(log_dir, logs_zip) clean_dir(log_dir, filter_=lambda p: p != logs_zip) if 'models' in artifacts: models_zip = os.path.join(output_dir, "models.zip") zip_path(output_dir, models_zip) clean_dir(output_dir, filter_=lambda p: p != models_zip) shutil.rmtree(tmpdir, ignore_errors=True)
def run(dataset, config): log.info("\n**** GAMA [v%s] ****", __version__) log.info("sklearn == %s", sklearn.__version__) log.info("category_encoders == %s", category_encoders.__version__) is_classification = (config.type == 'classification') # Mapping of benchmark metrics to GAMA metrics metrics_mapping = dict( acc='accuracy', auc='roc_auc', f1='f1', logloss='neg_log_loss', mae='neg_mean_absolute_error', mse='neg_mean_squared_error', msle='neg_mean_squared_log_error', r2='r2', rmse='neg_mean_squared_error', ) scoring_metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else None if scoring_metric is None: raise ValueError("Performance metric {} not supported.".format( config.metric)) training_params = { k: v for k, v in config.framework_params.items() if not k.startswith('_') } n_jobs = config.framework_params.get( '_n_jobs', config.cores ) # useful to disable multicore, regardless of the dataset config log.info( 'Running GAMA with a maximum time of %ss on %s cores, optimizing %s.', config.max_runtime_seconds, n_jobs, scoring_metric) estimator = GamaClassifier if is_classification else GamaRegressor kwargs = dict(n_jobs=n_jobs, max_total_time=config.max_runtime_seconds, scoring=scoring_metric, random_state=config.seed, **training_params) version_leq_20_2_0 = version.parse(__version__) <= version.parse('20.2.0') if version_leq_20_2_0: log_file = touch( os.path.join(output_subdir('logs', config), 'gama.log')) kwargs['keep_analysis_log'] = log_file else: kwargs['max_memory_mb'] = config.max_mem_size_mb kwargs['output_directory'] = output_subdir('logs', config) gama_automl = estimator(**kwargs) X_train, y_train = dataset.train.X, dataset.train.y # data = file_to_pandas(dataset.train.path, encoding='utf-8') # X_train, y_train = data.loc[:, data.columns != dataset.target], data.loc[:, dataset.target] with Timer() as training_timer: gama_automl.fit(X_train, y_train) log.info('Predicting on the test set.') X_test, y_test = dataset.test.X, dataset.test.y # data = file_to_pandas(dataset.test.path, encoding='utf-8') # X_test, y_test = data.loc[:, data.columns != dataset.target], data.loc[:, dataset.target] with Timer() as predict_timer: predictions = gama_automl.predict(X_test) if is_classification: probabilities = gama_automl.predict_proba(X_test) else: probabilities = None return result(output_file=config.output_predictions_file, predictions=predictions, probabilities=probabilities, truth=y_test, target_is_encoded=False, models_count=len(gama_automl._final_pop), training_duration=training_timer.duration, predict_duration=predict_timer.duration)
def run(dataset, config): log.info(f"\n**** H2O AutoML [v{h2o.__version__}] ****\n") save_metadata(config, version=h2o.__version__) # Mapping of benchmark metrics to H2O metrics metrics_mapping = dict(acc='mean_per_class_error', auc='AUC', logloss='logloss', mae='mae', mse='mse', r2='r2', rmse='rmse', rmsle='rmsle') sort_metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else None if sort_metric is None: # TODO: Figure out if we are going to blindly pass metrics through, or if we use a strict mapping log.warning("Performance metric %s not supported, defaulting to AUTO.", config.metric) try: training_params = { k: v for k, v in config.framework_params.items() if not k.startswith('_') } nthreads = config.framework_params.get('_nthreads', config.cores) jvm_memory = str( round(config.max_mem_size_mb * 2 / 3)) + "M" # leaving 1/3rd of available memory for XGBoost log.info("Starting H2O cluster with %s cores, %s memory.", nthreads, jvm_memory) max_port_range = 49151 min_port_range = 1024 rnd_port = os.getpid() % (max_port_range - min_port_range) + min_port_range port = config.framework_params.get('_port', rnd_port) init_params = config.framework_params.get('_init', {}) if "logs" in config.framework_params.get('_save_artifacts', []): init_params['ice_root'] = output_subdir("logs", config) h2o.init(nthreads=nthreads, port=port, min_mem_size=jvm_memory, max_mem_size=jvm_memory, **init_params) import_kwargs = {} # Load train as an H2O Frame, but test as a Pandas DataFrame log.debug("Loading train data from %s.", dataset.train.path) train = None if version.parse(h2o.__version__) >= version.parse( "3.32.0.3" ): # previous versions may fail to parse correctly some rare arff files using single quotes as enum/string delimiters (pandas also fails on same datasets) import_kwargs['quotechar'] = '"' train = h2o.import_file(dataset.train.path, destination_frame=frame_name( 'train', config), **import_kwargs) if not verify_loaded_frame(train, dataset): h2o.remove(train) train = None import_kwargs['quotechar'] = "'" if not train: train = h2o.import_file(dataset.train.path, destination_frame=frame_name( 'train', config), **import_kwargs) # train.impute(method='mean') log.debug("Loading test data from %s.", dataset.test.path) test = h2o.import_file(dataset.test.path, destination_frame=frame_name('test', config), **import_kwargs) # test.impute(method='mean') log.info("Running model on task %s, fold %s.", config.name, config.fold) log.debug( "Running H2O AutoML with a maximum time of %ss on %s core(s), optimizing %s.", config.max_runtime_seconds, config.cores, sort_metric) aml = H2OAutoML(max_runtime_secs=config.max_runtime_seconds, sort_metric=sort_metric, seed=config.seed, **training_params) monitor = ( BackendMemoryMonitoring( frequency_seconds=config.ext.monitoring.frequency_seconds, check_on_exit=True, verbosity=config.ext.monitoring.verbosity) if config.framework_params.get('_monitor_backend', False) # else contextlib.nullcontext # Py 3.7+ only else contextlib.contextmanager(iter)([0])) with utils.Timer() as training: with monitor: aml.train(y=dataset.target.index, training_frame=train) if not aml.leader: raise FrameworkError( "H2O could not produce any model in the requested time.") with utils.Timer() as predict: preds = aml.predict(test) preds = extract_preds(preds, test, dataset=dataset) save_artifacts(aml, dataset=dataset, config=config) return result(output_file=config.output_predictions_file, predictions=preds.predictions, truth=preds.truth, probabilities=preds.probabilities, probabilities_labels=preds.probabilities_labels, models_count=len(aml.leaderboard), training_duration=training.duration, predict_duration=predict.duration) finally: if h2o.connection(): # h2o.remove_all() h2o.connection().close() if h2o.connection().local_server: h2o.connection().local_server.shutdown()
def run(dataset, config): log.info(f"\n**** mljar-supervised [v{supervised.__version__}] ****\n") save_metadata(config, version=supervised.__version__) # Mapping of benchmark metrics to MLJAR metrics metrics_mapping = dict(auc='auc', logloss='logloss', rmse='rmse') eval_metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else "auto" # Mapping of benchmark task to MLJAR ML task problem_mapping = dict( binary="binary_classification", multiclass="multiclass_classification", regression="regression", ) ml_task = problem_mapping.get( dataset.problem_type ) # if None the AutoML will guess about the ML task is_classification = config.type == "classification" results_path = output_subdir("results", config) training_params = { k: v for k, v in config.framework_params.items() if not k.startswith("_") } column_names, _ = zip(*dataset.columns) column_types = dict(dataset.columns) label = dataset.target.name train = pd.DataFrame(dataset.train.data, columns=column_names).astype(column_types, copy=False) X_train = train.drop(columns=label) y_train = train[label] test = pd.DataFrame(dataset.test.data, columns=column_names).astype(column_types, copy=False) X_test = test.drop(columns=label) y_test = test[label] automl = AutoML(results_path=results_path, total_time_limit=config.max_runtime_seconds, random_state=config.seed, ml_task=ml_task, eval_metric=eval_metric, **training_params) with utils.Timer() as training: automl.fit(X_train, y_train) with utils.Timer() as predict: preds = automl.predict_all(X_test) predictions, probabilities = None, None if is_classification: predictions = preds["label"].values cols = [f"prediction_{c}" for c in np.unique(y_train)] probabilities = preds[cols].values else: predictions = preds["prediction"].values # clean the results if not config.framework_params.get("_save_artifacts", False): shutil.rmtree(results_path, ignore_errors=True) return result(output_file=config.output_predictions_file, predictions=predictions, truth=y_test, probabilities=probabilities, models_count=len(automl._models), training_duration=training.duration, predict_duration=predict.duration)
def save_artifacts(automl, dataset, config): artifacts = config.framework_params.get('_save_artifacts', ['leaderboard']) try: models_artifacts = [] lb = automl.leaderboard.as_data_frame() log.debug("Leaderboard:\n%s", lb.to_string()) if 'leaderboard' in artifacts: models_dir = output_subdir("models", config) lb_path = os.path.join(models_dir, "leaderboard.csv") write_csv(lb, lb_path) models_artifacts.append(lb_path) models_pat = re.compile(r"models(\[(json|binary|mojo)(?:,(\d+))?\])?") models = list(filter(models_pat.fullmatch, artifacts)) for m in models: models_dir = output_subdir("models", config) all_models_se = next( (mid for mid in lb['model_id'] if mid.startswith("StackedEnsemble_AllModels")), None) match = models_pat.fullmatch(m) mformat = match.group(2) or 'json' topN = int(match.group(3) or -1) if topN < 0 and mformat != 'json' and all_models_se: models_artifacts.append( save_model(all_models_se, dest_dir=models_dir, mformat=mformat)) else: count = 0 for mid in lb['model_id']: if topN < 0 or count < topN: save_model(mid, dest_dir=models_dir, mformat=mformat) count += 1 else: break models_archive = os.path.join(models_dir, f"models_{mformat}.zip") utils.zip_path(models_dir, models_archive, filtr=lambda p: p not in models_artifacts) models_artifacts.append(models_archive) def delete(path, isdir): if not isdir and path not in models_artifacts and os.path.splitext( path)[1] in ['.json', '.zip', '']: os.remove(path) utils.walk_apply(models_dir, delete, max_depth=0) if 'model_predictions' in artifacts: predictions_dir = output_subdir("predictions", config) test = h2o.get_frame(frame_name('test', config)) for mid in lb['model_id']: model = h2o.get_model(mid) h2o_preds = model.predict(test) preds = extract_preds(h2o_preds, test, dataset=dataset) if preds.probabilities_labels is None: preds.probabilities_labels = preds.h2o_labels write_preds( preds, os.path.join(predictions_dir, mid, 'predictions.csv')) utils.zip_path( predictions_dir, os.path.join(predictions_dir, "model_predictions.zip")) def delete(path, isdir): if isdir: shutil.rmtree(path, ignore_errors=True) utils.walk_apply(predictions_dir, delete, max_depth=0) if 'logs' in artifacts: logs_dir = output_subdir("logs", config) logs_zip = os.path.join(logs_dir, "h2o_logs.zip") utils.zip_path(logs_dir, logs_zip) # h2o.download_all_logs(dirname=logs_dir) def delete(path, isdir): if isdir: shutil.rmtree(path, ignore_errors=True) elif path != logs_zip: os.remove(path) utils.walk_apply(logs_dir, delete, max_depth=0) except Exception: log.debug("Error when saving artifacts.", exc_info=True)
def save_artifacts(automl, dataset, config): artifacts = config.framework_params.get('_save_artifacts', ['leaderboard']) try: models_artifacts = [] lb_pat = re.compile(r"leaderboard(?:\[(.*)\])?") lb_match = next((lb_pat.fullmatch(a) for a in artifacts), None) if lb_match: lb_ext = list( filter(None, re.split("[,; ]", (lb_match.group(1) or "")))) lb = h2o.automl.get_leaderboard(automl, lb_ext).as_data_frame() models_dir = output_subdir("models", config) lb_path = os.path.join(models_dir, "leaderboard.csv") write_csv(lb, lb_path) models_artifacts.append(lb_path) else: lb = automl.leaderboard.as_data_frame() log.debug("Leaderboard:\n%s", lb.to_string()) models_pat = re.compile(r"models(\[(json|binary|mojo)(?:,(\d+))?\])?") models = list(filter(models_pat.fullmatch, artifacts)) for m in models: models_dir = output_subdir("models", config) all_models_se = next( (mid for mid in lb['model_id'] if mid.startswith("StackedEnsemble_AllModels")), None) match = models_pat.fullmatch(m) mformat = match.group(2) or 'json' topN = int(match.group(3) or -1) if topN < 0 and mformat != 'json' and all_models_se: models_artifacts.append( save_model(all_models_se, dest_dir=models_dir, mformat=mformat)) else: count = 0 for mid in lb['model_id']: if topN < 0 or count < topN: save_model(mid, dest_dir=models_dir, mformat=mformat) count += 1 else: break models_archive = os.path.join(models_dir, f"models_{mformat}.zip") zip_path(models_dir, models_archive, filter_=lambda p: p not in models_artifacts) models_artifacts.append(models_archive) clean_dir(models_dir, filter_=lambda p: p not in models_artifacts and os. path.splitext(p)[1] in ['.json', '.zip', '']) if 'model_predictions' in artifacts: predictions_dir = output_subdir("predictions", config) test = h2o.get_frame(frame_name('test', config)) for mid in lb['model_id']: model = h2o.get_model(mid) h2o_preds = model.predict(test) preds = extract_preds(h2o_preds, test, dataset=dataset) if preds.probabilities_labels is None: preds.probabilities_labels = preds.h2o_labels write_preds( preds, os.path.join(predictions_dir, mid, 'predictions.csv')) predictions_zip = os.path.join(predictions_dir, "model_predictions.zip") zip_path(predictions_dir, predictions_zip) clean_dir(predictions_dir, filter_=lambda p: os.path.isdir(p)) if 'logs' in artifacts: logs_dir = output_subdir("logs", config) logs_zip = os.path.join(logs_dir, "h2o_logs.zip") zip_path(logs_dir, logs_zip) # h2o.download_all_logs(dirname=logs_dir) clean_dir(logs_dir, filter_=lambda p: p != logs_zip) write_csv(automl.event_log.as_data_frame(), os.path.join(logs_dir, 'aml_event_log.csv')) except Exception: log.debug("Error when saving artifacts.", exc_info=True)
def run(dataset, config): log.info("\n**** genens ****\n") is_classification = config.type == 'classification' if not is_classification: Warning("Regression not supported.") return None # Mapping of benchmark metrics to TPOT metrics metrics_mapping = { 'acc': get_scorer('accuracy'), 'auc': get_scorer('roc_auc'), 'f1': get_scorer('f1'), 'logloss': get_scorer('neg_log_loss'), 'mae': get_scorer('neg_mean_absolute_error'), 'mse': get_scorer('neg_mean_squared_error'), 'msle': get_scorer('neg_mean_squared_log_error'), 'r2': get_scorer('r2') } scoring_metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else None if scoring_metric is None: raise ValueError("Performance metric {} not supported.".format( config.metric)) X_train, X_test = dataset.train.X_enc, dataset.test.X_enc y_train, y_test = dataset.train.y_enc, dataset.test.y_enc training_params = { k: v for k, v in config.framework_params.items() if not k.startswith('_') } n_jobs = config.framework_params.get( '_n_jobs', config.cores ) # useful to disable multicore, regardless of the dataset config sample_size = config.framework_params.get('_sample_size', None) if sample_size is not None: evaluator = SampleCrossValEvaluator(sample_size=sample_size, per_gen=True, cv_k=5) else: evaluator = CrossValEvaluator(cv_k=5) print(f"Chosen sample size: {sample_size}.") print(f'cv_k: {evaluator.cv_k}') training_params['evaluator'] = evaluator runtime_s = config.max_runtime_seconds if runtime_s >= 600: runtime_s -= 5 * 60 # avoid premature process termination elif runtime_s > 10: runtime_s -= 5 if not config.framework_params.get('disable_logging', True): log_path = os.path.join(output_subdir('logs', config), 'evo_log_file.txt') else: log_path = None print(f"Setting time limit to {runtime_s} seconds.") log.info( 'Running genens with a maximum time of %ss on %s cores, optimizing %s.', runtime_s, n_jobs, scoring_metric) if config.seed is not None: # random state is yet to be unified in genens np.random.seed(config.seed) random.seed(config.seed) print(f'Training params: {training_params}') estimator = GenensClassifier if is_classification else GenensRegressor genens_est = estimator(n_jobs=n_jobs, max_evo_seconds=runtime_s, scorer=scoring_metric, log_path=log_path, **training_params) with utils.Timer() as training: genens_est.fit(X_train, y_train) log.info('Predicting on the test set.') best_pipe = genens_est.get_best_pipelines()[0] best_pipe.fit(X_train, y_train) predictions = best_pipe.predict(X_test) try: probabilities = best_pipe.predict_proba( X_test) if is_classification else None except AttributeError: target_values_enc = dataset.target.label_encoder.transform( dataset.target.values) probabilities = utils.Encoder( 'one-hot', target=False, encoded_type=float).fit(target_values_enc).transform(predictions) save_artifacts(genens_est, config) return result(output_file=config.output_predictions_file, predictions=predictions, truth=y_test, probabilities=probabilities, target_is_encoded=is_classification, models_count=len(genens_est.get_best_pipelines()), training_duration=training.duration)
def run(dataset, config): log.info(f"\n**** AutoGluon [v{__version__}] ****\n") save_metadata(config, version=__version__) metrics_mapping = dict( acc=metrics.accuracy, auc=metrics.roc_auc, f1=metrics.f1, logloss=metrics.log_loss, mae=metrics.mean_absolute_error, mse=metrics.mean_squared_error, r2=metrics.r2, rmse=metrics.root_mean_squared_error, ) label = dataset.target.name problem_type = dataset.problem_type perf_metric = metrics_mapping[config.metric] if config.metric in metrics_mapping else None if perf_metric is None: # TODO: figure out if we are going to blindly pass metrics through, or if we use a strict mapping log.warning("Performance metric %s not supported.", config.metric) is_classification = config.type == 'classification' training_params = {k: v for k, v in config.framework_params.items() if not k.startswith('_')} load_raw = config.framework_params.get('_load_raw', False) if load_raw: train, test = load_data_raw(dataset=dataset) else: column_names, _ = zip(*dataset.columns) column_types = dict(dataset.columns) train = pd.DataFrame(dataset.train.data, columns=column_names).astype(column_types, copy=False) print(f"Columns dtypes:\n{train.dtypes}") test = pd.DataFrame(dataset.test.data, columns=column_names).astype(column_types, copy=False) del dataset gc.collect() output_dir = output_subdir("models", config) with utils.Timer() as training: predictor = TabularPredictor( label=label, eval_metric=perf_metric.name, path=output_dir, problem_type=problem_type, ).fit( train_data=train, time_limit=config.max_runtime_seconds, **training_params ) del train y_test = test[label] test = test.drop(columns=label) if is_classification: with utils.Timer() as predict: probabilities = predictor.predict_proba(test, as_multiclass=True) predictions = probabilities.idxmax(axis=1).to_numpy() else: with utils.Timer() as predict: predictions = predictor.predict(test, as_pandas=False) probabilities = None prob_labels = probabilities.columns.values.tolist() if probabilities is not None else None leaderboard = predictor.leaderboard(silent=True) # Removed test data input to avoid long running computation, remove 7200s timeout limitation to re-enable with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 1000): print(leaderboard) save_artifacts(predictor, leaderboard, config) num_models_trained = len(leaderboard) if predictor._trainer.model_best is not None: num_models_ensemble = len(predictor._trainer.get_minimum_model_set(predictor._trainer.model_best)) else: num_models_ensemble = 1 return result(output_file=config.output_predictions_file, predictions=predictions, truth=y_test, probabilities=probabilities, probabilities_labels=prob_labels, target_is_encoded=False, models_count=num_models_trained, models_ensemble_count=num_models_ensemble, training_duration=training.duration, predict_duration=predict.duration)