def test_debug_container():
# Test if the debug option works. Check if some debug output from the server is visible.
set_log_level(True)
from hpobench.container.benchmarks.ml.xgboost_benchmark_old import XGBoostBenchmark as Benchmark
from hpobench.util.openml_data_manager import get_openmlcc18_taskids
task_id = get_openmlcc18_taskids()[0]
b = Benchmark(task_id=task_id,
container_name='xgboost_benchmark',
container_source='library://phmueller/automl')
cs = b.get_configuration_space()
assert cs is not None
set_log_level(False)
def run_experiment(on_travis: bool = False):
task_ids = get_openmlcc18_taskids()
for task_no, task_id in enumerate(task_ids):
if on_travis and task_no == 5:
break
print(
f'# ################### TASK {task_no + 1} of {len(task_ids)}: Task-Id: {task_id} ################### #'
)
if task_id == 167204:
continue # due to memory limits
b = Benchmark(task_id=task_id,
container_name='xgboost_benchmark',
container_source='library://phmueller/automl')
cs = b.get_configuration_space()
start = time()
num_configs = 1
for i in range(num_configs):
configuration = cs.sample_configuration()
print(configuration)
for n_estimator in [8, 64]:
for subsample in [0.4, 1]:
fidelity = {
'n_estimators': n_estimator,
'dataset_fraction': subsample
}
result_dict = b.objective_function(
configuration.get_dictionary(), fidelity=fidelity)
valid_loss = result_dict['function_value']
train_loss = result_dict['info']['train_loss']
assert result_dict['info']['fidelity'] == fidelity
result_dict = b.objective_function_test(
configuration, fidelity={'n_estimators': n_estimator})
test_loss = result_dict['function_value']
print(
f'[{i+1}|{num_configs}] No Estimator: {n_estimator:3d} - '
f'Subsample Rate: {subsample:.1f} - Test {test_loss:.4f} '
f'- Valid {valid_loss:.4f} - Train {train_loss:.4f}')
b.__del__()
print(f'Done, took totally {time()-start:.2f}')
import pytest
from hpobench.container.benchmarks.ml.svm_benchmark import SupportVectorMachine
from hpobench.util.openml_data_manager import get_openmlcc18_taskids
task_ids = get_openmlcc18_taskids()
import logging
logging.basicConfig(level=logging.DEBUG)
def test_svm_init():
benchmark = SupportVectorMachine(task_id=task_ids[0])
fs = benchmark.get_fidelity_space(seed=0)
fidelity = fs.sample_configuration().get_dictionary()
assert fidelity['dataset_fraction'] == pytest.approx(0.54881, abs=0.001)
meta = benchmark.get_meta_information()
assert meta is not None
cs = benchmark.get_configuration_space(seed=0)
config = cs.sample_configuration().get_dictionary()
assert config['C'] == pytest.approx(0.9762, abs=0.001)
assert config['gamma'] == pytest.approx(4.3037, abs=0.001)
result = benchmark.objective_function(configuration=config, fidelity=fidelity)
assert result['function_value'] == pytest.approx(0.4837, abs=0.1)
assert result['cost'] == pytest.approx(0.323, abs=0.1)
with pytest.raises(AssertionError):