def benchmark_best_model(args): task_type = STRING_TO_TASK_TYPES[args.task_type] path = args.data_path dataset = get_datasetname(path) metric = CLASSIFICATION_METRICS[args.metric] name = "SINGLE BEST" X, y, X_test, y_test = load_data(path) n_basemodels = X.shape[0] best_model = np.argmax(calculate_score(y, X[m,:,:], task_type, metric) for m in range(n_basemodels)) perf = calculate_score(y_test, X_test[best_model, :, :], task_type, metric) return {dataset: {name: {str(metric): perf}}}
def benchmark_stacking_agnostic_bayesian(args): from ensemble.stacking.stacking_agnostic_bayesian import AgnosticBayesian task_type = STRING_TO_TASK_TYPES[args.task_type] path = args.data_path name = "StackingAgnosticBayesian" metric = CLASSIFICATION_METRICS[args.metric] X, y, X_test, y_test = load_data(path) model = AgnosticBayesian(task_type, metric, n_bootstraps=500) return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_stacking_bayesian_avg_mcmc(args): from ensemble.stacking.stacking_bayesian_avg_mcmc import BayesianAverageMCMC task_type = STRING_TO_TASK_TYPES[args.task_type] path = args.data_path name = "StackingBayesianAverageMCMC" metric = CLASSIFICATION_METRICS[args.metric] X, y, X_test, y_test = load_data(path) model = BayesianAverageMCMC(n_samples=10000) return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_stacking_bayesian_avg(args): from ensemble.stacking.stacking_bayesian_avg import BayesianAverage task_type = STRING_TO_TASK_TYPES[args.task_type] path = args.data_path name = "StackingBayesianAverage" metric = CLASSIFICATION_METRICS[args.metric] X, y, X_test, y_test = load_data(path) if args.multi: model = BayesianAverage(multi=True) name = "StackingBayesianAverageMultiClass" model = BayesianAverage() name = "StackingBayesianAverageBinaryClass" return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_stacking_mlr(args): from ensemble.stacking.stacking_MLR import StackingMLR task_type = STRING_TO_TASK_TYPES[args.task_type] path = args.data_path name = "StackingMLR" metric = CLASSIFICATION_METRICS[args.metric] X, y, X_test, y_test = load_data(path) model = StackingMLR() return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_stacking_logit_reg(args): from ensemble.stacking.stacking_logit_reg import StackingLogitReg task_type = STRING_TO_TASK_TYPES[args.task_type] path = args.data_path name = "StackingLogitReg" metric = CLASSIFICATION_METRICS[args.metric] X, y, X_test, y_test = load_data(path) if args.reg_l1: model = StackingLogitReg(regularizer="l1") name = "StackingLogitRegularized_l1" model = StackingLogitReg(regularizer="l2") name = "StackingLogitRegularized_l2" return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_ensemble_selection(args): from ensemble.ensemble_selection.ensemble_selection import EnsembleSelection task_type = STRING_TO_TASK_TYPES[args.task_type] path = args.data_path metric = CLASSIFICATION_METRICS[args.metric] X, y, X_test, y_test = load_data(path) print(X.shape) if args.n_init and args.replacements: n_init = int(args.n_init) model = EnsembleSelection(task_type, metric, no_iterations=1000, sorted_initialization=True, with_replacements=True, n_init=n_init) name = "EnsembleSelectionSortedInitialization_WithReplacements" elif args.n_init: n_init = int(args.n_init) model = EnsembleSelection(task_type, metric, no_iterations=1000, sorted_initialization=True, n_init=n_init) name = "EnsembleSelectionSortedInitialization" elif args.replacements: model = EnsembleSelection(task_type, metric, no_iterations=1000, with_replacements=True) name = "EnsembleSelectionReplacements" else: model = EnsembleSelection(task_type, metric, no_iterations=1000) name = "EnsembleSelection" return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))