def _get_benchmark(dataset_name, data_dir):
benchmarks = dict(
protein=FCNetProteinStructureBenchmark(data_dir=data_dir),
slice=FCNetSliceLocalizationBenchmark(data_dir=data_dir),
naval=FCNetNavalPropulsionBenchmark(data_dir=data_dir),
parkinsons=FCNetParkinsonsTelemonitoringBenchmark(data_dir=data_dir))
return benchmarks.get(dataset_name)
def __init__(self,
data_path: Union[Path, str,
None] = './fcnet_tabular_benchmarks/',
rng: Union[np.random.RandomState, int, None] = None,
**kwargs):
from tabular_benchmarks import FCNetParkinsonsTelemonitoringBenchmark
benchmark = FCNetParkinsonsTelemonitoringBenchmark(
data_dir=str(data_path))
super(ParkinsonsTelemonitoringBenchmark,
self).__init__(benchmark=benchmark,
data_path=data_path,
rng=rng,
**kwargs)
def __init__(self,
data_path: Union[Path, str, None] = None,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs):
from hpobench import config_file
data_path = Path(
data_path
) if data_path is not None else config_file.data_dir / 'fcnet_tabular_benchmarks'
from tabular_benchmarks import FCNetParkinsonsTelemonitoringBenchmark
benchmark = FCNetParkinsonsTelemonitoringBenchmark(
data_dir=str(data_path))
super(ParkinsonsTelemonitoringBenchmark,
self).__init__(benchmark=benchmark,
data_path=data_path,
rng=rng,
**kwargs)
def get_tabular_bm_with_budget(benchmark: str):
nas_full_path = f'{TABULAR_ROOT}'
fcnet_path = f'{TABULAR_ROOT}/fcnet_tabular_benchmarks'
if benchmark == "nas_cifar10a":
min_budget = 12
max_budget = 108
b = NASCifar10A(data_dir=nas_full_path)
elif benchmark == "nas_cifar10b":
b = NASCifar10B(data_dir=nas_full_path)
min_budget = 12
max_budget = 108
elif benchmark == "nas_cifar10c":
b = NASCifar10C(data_dir=nas_full_path)
min_budget = 12
max_budget = 108
elif benchmark == "protein_structure":
b = FCNetProteinStructureBenchmark(data_dir=fcnet_path)
min_budget = 3
max_budget = 100
elif benchmark == "slice_localization":
b = FCNetSliceLocalizationBenchmark(data_dir=fcnet_path)
min_budget = 3
max_budget = 100
elif benchmark == "naval_propulsion":
b = FCNetNavalPropulsionBenchmark(data_dir=fcnet_path)
min_budget = 3
max_budget = 100
elif benchmark == "parkinsons_telemonitoring":
b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=fcnet_path)
min_budget = 3
max_budget = 100
else:
raise ValueError(f'benchmark {benchmark} not recognized')
return b, min_budget, max_budget
b = NASCifar10B(data_dir=data_dir, multi_fidelity=True)
elif benchmark_type == "nas_cifar10c": # NAS-Bench-101
max_budget = 108
b = NASCifar10C(data_dir=data_dir, multi_fidelity=True)
elif benchmark_type == "protein_structure": # NAS-HPO-Bench
max_budget = 100
b = FCNetProteinStructureBenchmark(data_dir=data_dir)
elif benchmark_type == "slice_localization": # NAS-HPO-Bench
max_budget = 100
b = FCNetSliceLocalizationBenchmark(data_dir=data_dir)
elif benchmark_type == "naval_propulsion": # NAS-HPO-Bench
max_budget = 100
b = FCNetNavalPropulsionBenchmark(data_dir=data_dir)
else: # benchmark_type == "parkinsons_telemonitoring": # NAS-HPO-Bench
max_budget = 100
b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=data_dir)
def f(config, budget=None):
if budget is not None:
fitness, cost = b.objective_function(config, budget=int(budget))
else:
fitness, cost = b.objective_function(config)
return fitness, cost
cs = b.get_configuration_space()
dimensions = len(cs.get_hyperparameters())
elif benchmark == '1shot1':
assert benchmark_type in ['1', '2', '3']
sys.path.append(os.path.join(os.getcwd(), '../nasbench/'))
def setUp(self):
self.b = FCNetParkinsonsTelemonitoringBenchmark(
data_dir="./fcnet_tabular_benchmarks/")