def test_benchmarks_executor():
train_file, test_file = get_cancer_case_data_paths()
result_metrics = CaseExecutor(
params=ExecutionParams(train_file=train_file,
test_file=test_file,
task=TaskTypesEnum.classification,
case_label='cancer',
target_name='target'),
models=[
BenchmarkModelTypesEnum.fedot, BenchmarkModelTypesEnum.baseline
],
metric_list=['roc_auc', 'f1']).execute()
assert result_metrics is not None
from benchmark.benchmark_model_types import BenchmarkModelTypesEnum
from benchmark.benchmark_utils import get_models_hyperparameters, get_scoring_case_data_paths, \
save_metrics_result_file
from benchmark.executor import CaseExecutor, ExecutionParams
from core.repository.tasks import TaskTypesEnum
if __name__ == '__main__':
train_file, test_file = get_scoring_case_data_paths()
result_metrics = CaseExecutor(
params=ExecutionParams(train_file=train_file,
test_file=test_file,
task=TaskTypesEnum.classification,
target_name='default',
case_label='scoring'),
models=[
BenchmarkModelTypesEnum.baseline, BenchmarkModelTypesEnum.tpot,
BenchmarkModelTypesEnum.fedot
],
metric_list=['roc_auc', 'f1']).execute()
result_metrics['hyperparameters'] = get_models_hyperparameters()
save_metrics_result_file(result_metrics, file_name='scoring_metrics')
name_of_dataset, num_classes)
if not problem_class or not metric_names:
print('Incorrect dataset')
continue
train_file, test_file = get_penn_case_data_paths(name_of_dataset)
config_models_data = get_models_hyperparameters()
case_name = f'penn_ml_{name_of_dataset}'
try:
result_metrics = CaseExecutor(params=ExecutionParams(
train_file=train_file,
test_file=test_file,
task=problem_class,
target_name='target',
case_label=case_name),
models=[
BenchmarkModelTypesEnum.tpot,
BenchmarkModelTypesEnum.baseline,
BenchmarkModelTypesEnum.fedot
],
metric_list=metric_names).execute()
except Exception as ex:
print(f'Exception on {name_of_dataset}: {ex}')
continue
result_metrics['hyperparameters'] = config_models_data
save_metrics_result_file(
result_metrics, file_name=f'penn_ml_metrics_for_{name_of_dataset}')
convert_json_stats_to_csv(dataset)
from benchmark.benchmark_model_types import BenchmarkModelTypesEnum
from benchmark.benchmark_utils import get_cancer_case_data_paths, save_metrics_result_file, get_models_hyperparameters
from benchmark.executor import CaseExecutor
from core.repository.task_types import MachineLearningTasksEnum
if __name__ == '__main__':
train_file, test_file = get_cancer_case_data_paths()
result_metrics = CaseExecutor(train_file=train_file,
test_file=test_file,
task=MachineLearningTasksEnum.classification,
case_label='cancer',
target_name='target',
models=[
BenchmarkModelTypesEnum.tpot,
BenchmarkModelTypesEnum.h2o,
BenchmarkModelTypesEnum.autokeras,
BenchmarkModelTypesEnum.mlbox,
BenchmarkModelTypesEnum.baseline
]).execute()
result_metrics['hyperparameters'] = get_models_hyperparameters()
save_metrics_result_file(result_metrics, file_name='cancer_metrics')
from benchmark.benchmark_model_types import BenchmarkModelTypesEnum
from benchmark.benchmark_utils import get_cancer_case_data_paths, get_models_hyperparameters, save_metrics_result_file
from benchmark.executor import CaseExecutor, ExecutionParams
from core.repository.tasks import TaskTypesEnum
if __name__ == '__main__':
train_file, test_file = get_cancer_case_data_paths()
result_metrics = CaseExecutor(params=ExecutionParams(train_file=train_file,
test_file=test_file,
task=TaskTypesEnum.classification,
case_label='cancer',
target_name='target'),
models=[BenchmarkModelTypesEnum.tpot,
BenchmarkModelTypesEnum.h2o,
BenchmarkModelTypesEnum.autokeras,
BenchmarkModelTypesEnum.mlbox,
BenchmarkModelTypesEnum.baseline],
metric_list=['roc_auc', 'f1']).execute()
result_metrics['hyperparameters'] = get_models_hyperparameters()
save_metrics_result_file(result_metrics, file_name='cancer_metrics')
from benchmark.benchmark_model_types import BenchmarkModelTypesEnum
from benchmark.benchmark_utils import get_scoring_case_data_paths, save_metrics_result_file, get_models_hyperparameters
from benchmark.executor import CaseExecutor
from core.repository.task_types import MachineLearningTasksEnum
if __name__ == '__main__':
train_file, test_file = get_scoring_case_data_paths()
result_metrics = CaseExecutor(train_file=train_file,
test_file=test_file,
task=MachineLearningTasksEnum.classification,
models=[BenchmarkModelTypesEnum.tpot,
BenchmarkModelTypesEnum.h2o,
BenchmarkModelTypesEnum.fedot,
BenchmarkModelTypesEnum.autokeras,
BenchmarkModelTypesEnum.mlbox],
target_name='default',
case_label='scoring').execute()
result_metrics['hyperparameters'] = get_models_hyperparameters()
save_metrics_result_file(result_metrics, file_name='scoring_metrics')