def benchmark_best_model(args):
task_type = STRING_TO_TASK_TYPES[args.task_type]
path = args.data_path
dataset = get_datasetname(path)
metric = CLASSIFICATION_METRICS[args.metric]
name = "SINGLE BEST"
X, y, X_test, y_test = load_data(path)
n_basemodels = X.shape[0]
best_model = np.argmax(calculate_score(y, X[m,:,:], task_type, metric) for m in range(n_basemodels))
perf = calculate_score(y_test, X_test[best_model, :, :], task_type, metric)
return {dataset: {name: {str(metric): perf}}}
def benchmark_stacking_agnostic_bayesian(args):
from ensemble.stacking.stacking_agnostic_bayesian import AgnosticBayesian
task_type = STRING_TO_TASK_TYPES[args.task_type]
path = args.data_path
name = "StackingAgnosticBayesian"
metric = CLASSIFICATION_METRICS[args.metric]
X, y, X_test, y_test = load_data(path)
model = AgnosticBayesian(task_type, metric, n_bootstraps=500)
return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_stacking_bayesian_avg_mcmc(args):
from ensemble.stacking.stacking_bayesian_avg_mcmc import BayesianAverageMCMC
task_type = STRING_TO_TASK_TYPES[args.task_type]
path = args.data_path
name = "StackingBayesianAverageMCMC"
metric = CLASSIFICATION_METRICS[args.metric]
X, y, X_test, y_test = load_data(path)
model = BayesianAverageMCMC(n_samples=10000)
return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_stacking_bayesian_avg(args):
from ensemble.stacking.stacking_bayesian_avg import BayesianAverage
task_type = STRING_TO_TASK_TYPES[args.task_type]
path = args.data_path
name = "StackingBayesianAverage"
metric = CLASSIFICATION_METRICS[args.metric]
X, y, X_test, y_test = load_data(path)
if args.multi:
model = BayesianAverage(multi=True)
name = "StackingBayesianAverageMultiClass"
model = BayesianAverage()
name = "StackingBayesianAverageBinaryClass"
return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_stacking_mlr(args):
from ensemble.stacking.stacking_MLR import StackingMLR
task_type = STRING_TO_TASK_TYPES[args.task_type]
path = args.data_path
name = "StackingMLR"
metric = CLASSIFICATION_METRICS[args.metric]
X, y, X_test, y_test = load_data(path)
model = StackingMLR()
return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_stacking_logit_reg(args):
from ensemble.stacking.stacking_logit_reg import StackingLogitReg
task_type = STRING_TO_TASK_TYPES[args.task_type]
path = args.data_path
name = "StackingLogitReg"
metric = CLASSIFICATION_METRICS[args.metric]
X, y, X_test, y_test = load_data(path)
if args.reg_l1:
model = StackingLogitReg(regularizer="l1")
name = "StackingLogitRegularized_l1"
model = StackingLogitReg(regularizer="l2")
name = "StackingLogitRegularized_l2"
return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))
def benchmark_ensemble_selection(args):
from ensemble.ensemble_selection.ensemble_selection import EnsembleSelection
task_type = STRING_TO_TASK_TYPES[args.task_type]
path = args.data_path
metric = CLASSIFICATION_METRICS[args.metric]
X, y, X_test, y_test = load_data(path)
print(X.shape)
if args.n_init and args.replacements:
n_init = int(args.n_init)
model = EnsembleSelection(task_type, metric, no_iterations=1000,
sorted_initialization=True, with_replacements=True,
n_init=n_init)
name = "EnsembleSelectionSortedInitialization_WithReplacements"
elif args.n_init:
n_init = int(args.n_init)
model = EnsembleSelection(task_type, metric, no_iterations=1000,
sorted_initialization=True, n_init=n_init)
name = "EnsembleSelectionSortedInitialization"
elif args.replacements:
model = EnsembleSelection(task_type, metric, no_iterations=1000,
with_replacements=True)
name = "EnsembleSelectionReplacements"
else:
model = EnsembleSelection(task_type, metric, no_iterations=1000)
name = "EnsembleSelection"
return apply_model(args, X=X, y=y, X_test=X_test, y_test=y_test, metric=metric, task_type=task_type, model=model, name=name, dataset=get_datasetname(path))