def test_whitebox_without_container_xgb():
from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark
b = Benchmark(task_id=167199, rng=0)
cs = b.get_configuration_space(seed=0)
configuration = cs.get_default_configuration()
assert configuration['colsample_bylevel'] == 1.0
assert len(configuration.keys()) == 6
n_estimator = 32
subsample = 1
result_dict = b.objective_function(configuration,
fidelity=dict(n_estimators=n_estimator,
subsample=subsample),
rng=0)
valid_loss = result_dict['function_value']
train_loss = result_dict['info']['train_loss']
result_dict = b.objective_function_test(
configuration, fidelity=dict(n_estimators=n_estimator), rng=0)
test_loss = result_dict['function_value']
assert np.isclose(train_loss, 0.1071, atol=0.001)
assert np.isclose(valid_loss, 0.3873, atol=0.001)
assert np.isclose(test_loss, 0.38181, atol=0.001)
def test_cartpole():
from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark
b = Benchmark(container_name='cartpole', rng=1)
cs = b.get_configuration_space(seed=1)
print(cs.get_default_configuration())
from hpobench.container.benchmarks.rl.cartpole import CartpoleFull as Benchmark
b = Benchmark(container_name='cartpole', rng=1)
cs = b.get_configuration_space(seed=1)
print(cs.get_default_configuration())
def test_rng_serialization_xgb():
import json
from hpobench.util.container_utils import BenchmarkEncoder, BenchmarkDecoder
from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark
b = XGBoostBenchmark(task_id=167149, rng=0)
meta = b.get_meta_information()
meta_str = json.dumps(meta, indent=None, cls=BenchmarkEncoder)
meta_new = json.loads(meta_str, cls=BenchmarkDecoder)
assert isinstance(meta_new['initial random seed'], np.random.RandomState)
assert np.array_equiv(meta['initial random seed'].random(10),
meta_new['initial random seed'].random(10))
def run_experiment(on_travis: bool = False):
task_ids = get_openmlcc18_taskids()
for task_no, task_id in enumerate(task_ids):
if on_travis and task_no == 5:
break
print(
f'# ################### TASK {task_no + 1} of {len(task_ids)}: Task-Id: {task_id} ################### #'
)
if task_id == 167204:
continue # due to memory limits
b = Benchmark(task_id=task_id)
cs = b.get_configuration_space()
start = time()
num_configs = 1
for i in range(num_configs):
configuration = cs.sample_configuration()
print(configuration)
for n_estimator in [8, 64]:
for subsample in [0.4, 1]:
fidelity = {
'n_estimators': n_estimator,
'dataset_fraction': subsample
}
result_dict = b.objective_function(
configuration.get_dictionary(), fidelity=fidelity)
valid_loss = result_dict['function_value']
train_loss = result_dict['info']['train_loss']
assert result_dict['info']['fidelity'] == fidelity
result_dict = b.objective_function_test(configuration)
test_loss = result_dict['function_value']
print(
f'[{i+1}|{num_configs}] No Estimator: {n_estimator:3d} - '
f'Subsample Rate: {subsample:.1f} - Test {test_loss:.4f} '
f'- Valid {valid_loss:.4f} - Train {train_loss:.4f}')
print(f'Done, took totally {time()-start:.2f}')
def test_whitebox_with_container():
from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark
b = Benchmark(container_name='xgboost_benchmark',
task_id=167199,
rng=0)
cs = b.get_configuration_space()
configuration = cs.get_default_configuration()
assert configuration['colsample_bylevel'] == 1.0
assert len(configuration.keys()) == 8
n_estimator = 32
subsample = 1
result_dict = b.objective_function(configuration, fidelity=dict(n_estimators=n_estimator,
dataset_fraction=subsample))
valid_loss = result_dict['function_value']
train_loss = result_dict['info']['train_loss']
result_dict = b.objective_function_test(configuration, fidelity=dict(n_estimators=n_estimator))
test_loss = result_dict['function_value']
assert np.isclose(train_loss, 0.02232, atol=0.001)
assert np.isclose(valid_loss, 0.4234, atol=0.001)
assert np.isclose(test_loss, 0.43636, atol=0.001)