class TestFCNetProteinStructure(unittest.TestCase):
def setUp(self):
self.b = FCNetProteinStructureBenchmark(
data_dir="./fcnet_tabular_benchmarks/")
def test_random_sampling(self):
config = self.b.get_configuration_space().sample_configuration()
self.b.objective_function(config)
def _get_benchmark(dataset_name, data_dir):
benchmarks = dict(
protein=FCNetProteinStructureBenchmark(data_dir=data_dir),
slice=FCNetSliceLocalizationBenchmark(data_dir=data_dir),
naval=FCNetNavalPropulsionBenchmark(data_dir=data_dir),
parkinsons=FCNetParkinsonsTelemonitoringBenchmark(data_dir=data_dir))
return benchmarks.get(dataset_name)
def __init__(self,
data_path: Union[Path, str,
None] = './fcnet_tabular_benchmarks/',
rng: Union[np.random.RandomState, int, None] = None,
**kwargs):
from tabular_benchmarks import FCNetProteinStructureBenchmark
benchmark = FCNetProteinStructureBenchmark(data_dir=str(data_path))
super(ProteinStructureBenchmark, self).__init__(benchmark=benchmark,
data_path=data_path,
rng=rng,
**kwargs)
def __init__(self,
data_path: Union[Path, str, None] = None,
rng: Union[np.random.RandomState, int, None] = None,
**kwargs):
from hpobench import config_file
data_path = Path(
data_path
) if data_path is not None else config_file.data_dir / 'fcnet_tabular_benchmarks'
from tabular_benchmarks import FCNetProteinStructureBenchmark
benchmark = FCNetProteinStructureBenchmark(data_dir=str(data_path))
super(ProteinStructureBenchmark, self).__init__(benchmark=benchmark,
data_path=data_path,
rng=rng,
**kwargs)
def get_tabular_bm_with_budget(benchmark: str):
nas_full_path = f'{TABULAR_ROOT}'
fcnet_path = f'{TABULAR_ROOT}/fcnet_tabular_benchmarks'
if benchmark == "nas_cifar10a":
min_budget = 12
max_budget = 108
b = NASCifar10A(data_dir=nas_full_path)
elif benchmark == "nas_cifar10b":
b = NASCifar10B(data_dir=nas_full_path)
min_budget = 12
max_budget = 108
elif benchmark == "nas_cifar10c":
b = NASCifar10C(data_dir=nas_full_path)
min_budget = 12
max_budget = 108
elif benchmark == "protein_structure":
b = FCNetProteinStructureBenchmark(data_dir=fcnet_path)
min_budget = 3
max_budget = 100
elif benchmark == "slice_localization":
b = FCNetSliceLocalizationBenchmark(data_dir=fcnet_path)
min_budget = 3
max_budget = 100
elif benchmark == "naval_propulsion":
b = FCNetNavalPropulsionBenchmark(data_dir=fcnet_path)
min_budget = 3
max_budget = 100
elif benchmark == "parkinsons_telemonitoring":
b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=fcnet_path)
min_budget = 3
max_budget = 100
else:
raise ValueError(f'benchmark {benchmark} not recognized')
return b, min_budget, max_budget
data_dir = os.path.join(
os.getcwd(), "../nas_benchmarks-development/"
"tabular_benchmarks/fcnet_tabular_benchmarks/")
if benchmark_type == "nas_cifar10a": # NAS-Bench-101
max_budget = 108
b = NASCifar10A(data_dir=data_dir, multi_fidelity=True)
elif benchmark_type == "nas_cifar10b": # NAS-Bench-101
max_budget = 108
b = NASCifar10B(data_dir=data_dir, multi_fidelity=True)
elif benchmark_type == "nas_cifar10c": # NAS-Bench-101
max_budget = 108
b = NASCifar10C(data_dir=data_dir, multi_fidelity=True)
elif benchmark_type == "protein_structure": # NAS-HPO-Bench
max_budget = 100
b = FCNetProteinStructureBenchmark(data_dir=data_dir)
elif benchmark_type == "slice_localization": # NAS-HPO-Bench
max_budget = 100
b = FCNetSliceLocalizationBenchmark(data_dir=data_dir)
elif benchmark_type == "naval_propulsion": # NAS-HPO-Bench
max_budget = 100
b = FCNetNavalPropulsionBenchmark(data_dir=data_dir)
else: # benchmark_type == "parkinsons_telemonitoring": # NAS-HPO-Bench
max_budget = 100
b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=data_dir)
def f(config, budget=None):
if budget is not None:
fitness, cost = b.objective_function(config, budget=int(budget))
else:
fitness, cost = b.objective_function(config)
min_budget = 4
max_budget = 108
b = NASCifar10A(data_dir=args.data_dir)
elif args.benchmark == "nas_cifar10b":
b = NASCifar10B(data_dir=args.data_dir)
min_budget = 4
max_budget = 108
elif args.benchmark == "nas_cifar10c":
b = NASCifar10C(data_dir=args.data_dir)
min_budget = 4
max_budget = 108
elif args.benchmark == "protein_structure":
b = FCNetProteinStructureBenchmark(data_dir=args.data_dir)
min_budget = 3
max_budget = 100
elif args.benchmark == "slice_localization":
b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir)
min_budget = 3
max_budget = 100
elif args.benchmark == "naval_propulsion":
b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir)
min_budget = 3
max_budget = 100
elif args.benchmark == "parkinsons_telemonitoring":
b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir)
b = NASCifar10A(data_dir=data_dir, multi_fidelity=False)
y_star_valid = b.y_star_valid
y_star_test = b.y_star_test
inc_config = None
elif ssp == "nas_cifar10b":
b = NASCifar10B(data_dir=data_dir, multi_fidelity=False)
y_star_valid = b.y_star_valid
y_star_test = b.y_star_test
inc_config = None
elif ssp == "nas_cifar10c":
b = NASCifar10C(data_dir=data_dir, multi_fidelity=False)
y_star_valid = b.y_star_valid
y_star_test = b.y_star_test
inc_config = None
elif ssp == "protein_structure":
b = FCNetProteinStructureBenchmark(data_dir=data_dir)
inc_config, y_star_valid, y_star_test = b.get_best_configuration()
elif ssp == "slice_localization":
b = FCNetSliceLocalizationBenchmark(data_dir=data_dir)
inc_config, y_star_valid, y_star_test = b.get_best_configuration()
elif ssp == "naval_propulsion":
b = FCNetNavalPropulsionBenchmark(data_dir=data_dir)
inc_config, y_star_valid, y_star_test = b.get_best_configuration()
elif ssp == "parkinsons_telemonitoring":
b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=data_dir)
inc_config, y_star_valid, y_star_test = b.get_best_configuration()
elif benchmark == '1shot1':
sys.path.append(os.path.join(os.getcwd(), '../nasbench/'))
sys.path.append(os.path.join(os.getcwd(), '../nasbench-1shot1/'))
from nasbench_analysis.search_spaces.search_space_1 import SearchSpace1
y_star_valid = b.y_star_valid
y_star_test = b.y_star_test
inc_config = None
elif args.benchmark == "nas_cifar10c": # NAS-Bench-101
min_budget = 4
max_budget = 108
b = NASCifar10C(data_dir=args.data_dir, multi_fidelity=True)
y_star_valid = b.y_star_valid
y_star_test = b.y_star_test
inc_config = None
elif args.benchmark == "protein_structure": # NAS-HPO-Bench
min_budget = 3
max_budget = 100
b = FCNetProteinStructureBenchmark(data_dir=args.data_dir)
inc_config, y_star_valid, y_star_test = b.get_best_configuration()
elif args.benchmark == "slice_localization": # NAS-HPO-Bench
min_budget = 3
max_budget = 100
b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir)
inc_config, y_star_valid, y_star_test = b.get_best_configuration()
elif args.benchmark == "naval_propulsion": # NAS-HPO-Bench
min_budget = 3
max_budget = 100
b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir)
inc_config, y_star_valid, y_star_test = b.get_best_configuration()
elif args.benchmark == "parkinsons_telemonitoring": # NAS-HPO-Bench
from tabular_benchmarks import FCNetProteinStructureBenchmark
b = FCNetProteinStructureBenchmark(data_dir="./fcnet_tabular_benchmarks/")
cs = b.get_configuration_space()
config = cs.sample_configuration()
print("Numpy representation: ", config.get_array())
print("Dict representation: ", config.get_dictionary())
max_epochs = 100
y, cost = b.objective_function(config, budget=max_epochs)
print(y, cost)
from ultraopt import fmin
from ultraopt.multi_fidelity import HyperBandIterGenerator
parser = argparse.ArgumentParser()
parser.add_argument('--run_id', default=0, type=int, nargs='?', help='unique number to identify this run')
parser.add_argument('--optimizer', default="ETPE", type=str, nargs='?', help='Which optimizer to use')
parser.add_argument('--benchmark', default="protein_structure", type=str, nargs='?', help='specifies the benchmark')
parser.add_argument('--n_iters', default=100, type=int, nargs='?', help='number of iterations for optimization method')
parser.add_argument('--output_path', default="./", type=str, nargs='?',
help='specifies the path where the results will be saved')
parser.add_argument('--data_dir', default="./", type=str, nargs='?', help='specifies the path to the tabular data')
args = parser.parse_args()
if args.benchmark == "protein_structure":
b = FCNetProteinStructureBenchmark(data_dir=args.data_dir)
elif args.benchmark == "slice_localization":
b = FCNetSliceLocalizationBenchmark(data_dir=args.data_dir)
elif args.benchmark == "naval_propulsion":
b = FCNetNavalPropulsionBenchmark(data_dir=args.data_dir)
elif args.benchmark == "parkinsons_telemonitoring":
b = FCNetParkinsonsTelemonitoringBenchmark(data_dir=args.data_dir)
else:
raise NotImplementedError
output_path = os.path.join(args.output_path, f"{args.benchmark}-ultraopt_{args.optimizer}")
os.makedirs(os.path.join(output_path), exist_ok=True)
def objective_function(config: dict, budget: int = 100):
loss, cost = b.objective_function(config, int(budget))
def setUp(self):
self.b = FCNetProteinStructureBenchmark(
data_dir="./fcnet_tabular_benchmarks/")