def get_tuned_config(self, scenario: ASlibScenario,
runcount_limit:int=42,
wallclock_limit:int=300,
autofolio_config:dict=dict(),
seed:int=42):
'''
uses SMAC3 to determine a well-performing configuration in the configuration space self.cs on the given scenario
Arguments
---------
scenario: ASlibScenario
ASlib Scenario at hand
runcount_limit: int
runcount_limit for SMAC scenario
wallclock_limit: int
wallclock limit in sec for SMAC scenario
(overwritten by autofolio_config)
autofolio_config: dict, or None
An optional dictionary of configuration options
seed: int
random seed for SMAC
Returns
-------
Configuration
best incumbent configuration found by SMAC
'''
wallclock_limit = autofolio_config.get("wallclock_limit", wallclock_limit)
runcount_limit = autofolio_config.get("runcount_limit", runcount_limit)
taf = functools.partial(self.called_by_smac, scenario=scenario)
max_fold = scenario.cv_data.max().max()
max_fold = int(max_fold)
ac_scenario = Scenario({"run_obj": "quality", # we optimize quality
"runcount-limit": runcount_limit,
"cs": self.cs, # configuration space
"deterministic": "true",
"instances": [[str(i)] for i in range(1, max_fold+1)],
"wallclock-limit": wallclock_limit,
"output-dir" : "" if not autofolio_config.get("output-dir",None) else autofolio_config.get("output-dir")
})
# necessary to use stats options related to scenario information
AC_Stats.scenario = ac_scenario
# Optimize
self.logger.info(
">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
self.logger.info("Start Configuration")
self.logger.info(
">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
smac = SMAC(scenario=ac_scenario, tae_runner=taf,
rng=np.random.RandomState(seed))
incumbent = smac.optimize()
self.logger.info("Final Incumbent: %s" % (incumbent))
return incumbent
def perform_optimization(self, optimization_time_budget):
def _evaluate_config(config):
vector = self._smac_dict_to_vector(config.get_dictionary())
score = self._score_candidate(vector)
return 1 - score
if not self._is_stop_event_set():
smac = SMAC4HPO(
scenario=self._create_scenario(optimization_time_budget),
tae_runner=_evaluate_config,
rng=self.numpy_random_state,
runhistory=self._create_run_history(),
)
try:
candidate = smac.optimize()
finally:
candidate = smac.solver.incumbent
vector = self._smac_dict_to_vector(candidate.get_dictionary())
score = self._score_candidate(vector)
if score > self.best_score:
self.best_score = score
self.best_candidate = self._smac_dict_to_vector(
candidate.get_dictionary())
return (
self.parameter_domain.config_from_vector(self.best_candidate),
self.best_score,
)
def test_exchange_sobol_for_lhd(self):
cs = ConfigurationSpace()
for i in range(40):
cs.add_hyperparameter(
UniformFloatHyperparameter('x%d' % (i + 1), 0, 1))
scenario = Scenario({'cs': cs, 'run_obj': 'quality'})
facade = SMAC4HPO(scenario=scenario)
self.assertIsInstance(facade.solver.initial_design, SobolDesign)
cs.add_hyperparameter(UniformFloatHyperparameter('x41', 0, 1))
with self.assertRaisesRegex(
ValueError,
'Sobol sequence" can only handle up to 40 dimensions. Please use a different initial design, such as '
'"the Latin Hypercube design"',
):
SMAC4HPO(scenario=scenario)
self.output_dirs.append(scenario.output_dir)
def optimize(self):
cs = ConfigurationSpace()
if 'optimizer' in self.params:
optimizer = CategoricalHyperparameter('optimizer',
self.params['optimizer'])
cs.add_hyperparameter(optimizer)
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": 5,
"cutoff-time": 10,
"cs": cs,
"deterministic": "true"
})
print(cs.get_default_configuration())
def_value = self.svm_from_config(cs.get_default_configuration())
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=self.svm_from_config)
incumbent = smac.optimize()
__params = incumbent.get_dictionary()
inc_value = self.svm_from_config(incumbent)
print(__params)
model = create_model(**__params)
hist = model.fit(self.x_train, self.y_train, batch_size=128, epochs=6)
time = smac.stats.wallclock_time_used
loss, accuracy, f1_score, precision, recall = model.evaluate(
self.x_test, self.y_test, verbose=0)
del smac
self.result = {
'accuracy': accuracy,
'time': time,
'best_params': __params
}
def main():
for synthetic_function_cls in synthetic_functions:
meta_info = synthetic_function_cls.get_meta_information()
if "num_function_evals" in meta_info:
max_iter = meta_info["num_function_evals"]
else:
max_iter = base_max_iter
# 构造超参空间
config_space = ConfigurationSpace()
config_space.generate_all_continuous_from_bounds(
synthetic_function_cls.get_meta_information()['bounds'])
synthetic_function = synthetic_function_cls()
# 定义目标函数
def evaluation(config: dict):
config = Configuration(config_space, values=config)
return synthetic_function.objective_function(config)["function_value"] - \
synthetic_function.get_meta_information()["f_opt"]
res = pd.DataFrame(columns=[f"trial-{i}" for i in range(repetitions)],
index=range(max_iter))
print(meta_info["name"])
for trial in range(repetitions):
random_state = base_random_state + 10 * trial
# Scenario object
scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
"runcount-limit": max_iter,
# max. number of function evaluations; for this example set to a low number
"cs": config_space, # configuration space
"deterministic": "true"
})
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(random_state),
tae_runner=evaluation,
initial_design_kwargs={"init_budget": 20})
incumbent = smac.optimize()
runhistory = smac.runhistory
configs = runhistory.get_all_configs()
losses = [runhistory.get_cost(config) for config in configs]
res[f"trial-{trial}"] = losses
print(min(losses))
res = raw2min(res)
m = res.mean(1)
s = res.std(1)
name = synthetic_function.get_meta_information()["name"]
final_result[name] = {"mean": m.tolist(), "std": s.tolist()}
Path(f"SMAC3.json").write_text(json.dumps(final_result))
def create_or_restore_smac(scenario_dict, rng, tae):
out_dir = path.join(scenario_dict['output_dir'], 'run_1')
if True or not isfile(path.join(out_dir, "traj_aclib2.json")):
# if some incomplete data lays arround, delete it completely
shutil.rmtree(out_dir, ignore_errors=True)
scenario = Scenario(scenario_dict)
smac = SMAC4HPO(scenario=scenario,
rng=rng,
tae_runner=tae,
initial_design=SobolDesign,
run_id=1)
else:
new_scenario = Scenario(scenario_dict)
rh_path = path.join(out_dir, "runhistory.json")
runhistory = RunHistory(aggregate_func=None)
runhistory.load_json(rh_path, new_scenario.cs)
# ... stats, ...
stats_path = path.join(out_dir, "stats.json")
stats = Stats(new_scenario)
stats.load(stats_path)
# ... and trajectory.
traj_path = path.join(out_dir, "traj_aclib2.json")
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_path,
cs=new_scenario.cs)
incumbent = trajectory[-1]["incumbent"]
# Now we can initialize SMAC with the recovered objects and restore the
# state where we left off. By providing stats and a restore_incumbent, SMAC
# automatically detects the intention of restoring a state.
smac = SMAC4HPO(scenario=new_scenario,
runhistory=runhistory,
stats=stats,
restore_incumbent=incumbent,
run_id=1)
print('restored smac from:', out_dir)
return smac
def func(x):
dir="/user/tqc/test_dir"
# Scenario object
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 50,
# max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true",
"shared_model": True,
"input_psmac_dirs": dir,
'output_dir': dir,
}, runtime='spark',
spark_config={'hdfs_url': 'http://0.0.0.0:50070'})
smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42),
tae_runner=svm_from_cfg)
return smac.optimize()
def fmin_smac_nopynisher(func, x0, bounds, maxfun, rng):
"""
Minimize a function using SMAC, but without pynisher, which doesn't work
well with benchmark_minimize_callable.
This function is based on SMAC's fmin_smac.
"""
cs = ConfigurationSpace()
tmplt = 'x{0:0' + str(len(str(len(bounds)))) + 'd}'
for idx, (lower_bound, upper_bound) in enumerate(bounds):
parameter = UniformFloatHyperparameter(
name=tmplt.format(idx + 1),
lower=lower_bound,
upper=upper_bound,
default_value=x0[idx],
)
cs.add_hyperparameter(parameter)
scenario_dict = {
"run_obj": "quality",
"cs": cs,
"deterministic": "true",
"initial_incumbent": "DEFAULT",
"runcount_limit": maxfun,
}
scenario = Scenario(scenario_dict)
def call_ta(config):
x = np.array(
[val for _, val in sorted(config.get_dictionary().items())],
dtype=np.float)
return func(x)
smac = SMAC4HPO(
scenario=scenario,
tae_runner=ExecuteTAFuncArray,
tae_runner_kwargs={
'ta': call_ta,
'use_pynisher': False
},
rng=rng,
initial_design=RandomConfigurations,
)
smac.optimize()
return
def best_hyperparams_smac():
iteration = 1
cs = ConfigurationSpace()
cs.add_hyperparameters(SVD_SMAC_SPACE.values())
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit":
100, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
smac = SMAC4HPO(
scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=_hyperopt,
)
smac.optimize()
def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations, input_dir, output_dir):
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict()
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
def objective(config, seed):
return benchmark.evaluate(config).value
for run_id in range(run_start, num_runs):
random_state = np.random.RandomState(run_id)
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": num_iterations,
"cs": benchmark.get_config_space(),
"deterministic": "true",
"output_dir": "foo/"
})
run_history = RunHistory()
smac = SMAC4HPO(scenario=scenario,
tae_runner=objective,
runhistory=run_history,
rng=random_state)
smac.optimize()
data = SMACLogs(run_history).to_frame()
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def optimize_smac(problem, max_evals, rand_evals=1, deterministic=False, log=None):
n = len(problem.vartype())
nlog10 = math.ceil(math.log10(n))
mon = Monitor(f"smac{'/det' if deterministic else ''}{'/ac' if n > 40 else ''}", problem, log=log)
def f(cfg):
xvec = np.array([cfg.get(f'v{varidx:0{nlog10}}') for varidx, t in enumerate(problem.vartype())])
mon.commit_start_eval()
r = float(problem.evaluate(xvec))
mon.commit_end_eval(xvec, r)
return r
cs = get_variables(problem)
sc = Scenario({
"run_obj": "quality",
"runcount-limit": max_evals,
"cs": cs,
"output_dir": None,
"limit_resources": False, # Limiting resources stops the Monitor from working...
"deterministic": deterministic
})
# smac = SMAC4HPO(scenario=sc, tae_runner=f)
if n <= 40:
smac = SMAC4HPO(scenario=sc, initial_design=RandomConfigurations, initial_design_kwargs={'init_budget': rand_evals}, tae_runner=f)
else:
smac = SMAC4AC(scenario=sc, initial_design=RandomConfigurations, initial_design_kwargs={'init_budget': rand_evals}, tae_runner=f)
mon.start()
result = smac.optimize()
mon.end()
# print(f"Best trial: {best_trial}")
solX = [result[k] for k in result]
# print(f"Best point: {solX}")
# Note, this runs the function again, just to compute the fitness again.
# solY = f(solX)
# We can also ask it from our evaluation monitor.
solY = mon.best_fitness
return solX, solY, mon
def smac_opt():
# Import ConfigSpace and different types of parameters
from smac.configspace import ConfigurationSpace
from ConfigSpace.hyperparameters import UniformFloatHyperparameter
# Import SMAC-utilities
from smac.scenario.scenario import Scenario
from smac.facade.smac_hpo_facade import SMAC4HPO
n_params = 5
def fun_to_optimize(x):
from jnius import autoclass
RunStrategy = autoclass('RunStrategy')
params = [x[f'x{i}'] for i in range(0, n_params)]
print(f'params:{params}')
ret = -RunStrategy.runStrategyWithConfiguration(params, 20)
print(ret)
return ret
cs = ConfigurationSpace()
hyper_params = [
UniformFloatHyperparameter(f"x{i}", 0, 100, default_value=1)
for i in range(0, n_params)
]
cs.add_hyperparameters(hyper_params)
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit":
999999, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "false"
})
# Optimize, using a SMAC-object
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=fun_to_optimize)
smac.optimize()
def __init__(self, scenario: typing.Union[ScenarioWithSavepoint, Scenario, ScenarioProperties], seed=1):
self.scenario = scenario
self.scenario.output_dir_for_this_run = os.path.join(self.scenario.output_dir, 'run_1')
# Create defaults
rh = None
stats = None
incumbent = None
if os.path.exists(self.scenario.output_dir_for_this_run):
rh, stats, incumbent = restore_state(self.scenario)
self.smac = SMAC4HPO(scenario=self.scenario,
rng=np.random.RandomState(seed),
runhistory=rh,
initial_design_kwargs=dict(n_configs_x_params=1),
stats=stats,
restore_incumbent=incumbent,
run_id=1,
smbo_class=IncrementalSMBO,
tae_runner=self)
def optim_smac(args):
tae = snake_from_config_wrapper("SMAC", args.num_opponents,
args.num_games_per_eval, args.timeout)
def_value = tae(cs.get_default_configuration(),
0,
num_opponents=args.num_opponents,
num_games_per_eval=args.num_games_per_eval,
timeout=args.timeout)
print(
f"Default Configuration evaluates to a win percentage of {(1 - def_value) * 100:.2f}%"
)
print(
f"Starting Opimization with walltime of {args.walltime/3600:.2f} hours..."
)
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": args.runcount_limit,
"cs": cs,
"deterministic": True,
"wallclock_limit": args.walltime,
"output_dir": args.output_dir,
"shared_model": args.n_jobs > 1,
"input_psmac_dirs": args.output_dir,
})
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(RANDOMSTATES[0]),
tae_runner=tae)
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
inc_value = tae(incumbent)
print(
f"Optimized Configuration {incumbent} evaluates to a win percentage of {(1 - inc_value) * 100:.2f}%"
)
def __init__(self,
configspace: ConfigurationSpace,
working_directory: str = '.'):
super().__init__(configspace)
self.working_directory = os.path.join(
working_directory, f'smac/{random.randint(0, 10000000):d}/')
smac_logger = logging.getLogger('smac')
logging.getLogger('smac').setLevel(logging.WARNING)
scenario = Scenario({
'run_obj': 'quality',
'deterministic': True,
'shared-model': False,
'cs': self.configspace,
'output_dir': self.working_directory
})
scenario.logger = smac_logger
self.smbo: SplitSMBO = SMAC4HPO(scenario=scenario,
smbo_class=SplitSMBO).solver
self.smbo.logger = smac_logger
self.smbo.start()
def quant_post_hpo(
executor,
place,
model_dir,
quantize_model_path,
train_sample_generator=None,
eval_sample_generator=None,
train_dataloader=None,
eval_dataloader=None,
eval_function=None,
model_filename=None,
params_filename=None,
save_model_filename='__model__',
save_params_filename='__params__',
scope=None,
quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"],
is_full_quantize=False,
weight_bits=8,
activation_bits=8,
weight_quantize_type=['channel_wise_abs_max'],
algo=["KL", "hist", "avg", "mse"],
bias_correct=[True, False],
hist_percent=[0.98, 0.999], ### uniform sample in list.
batch_size=[10, 30], ### uniform sample in list.
batch_num=[10, 30], ### uniform sample in list.
optimize_model=False,
is_use_cache_file=False,
cache_dir="./temp_post_training",
runcount_limit=30):
"""
The function utilizes static post training quantization method to
quantize the fp32 model. It uses calibrate data to calculate the
scale factor of quantized variables, and inserts fake quantization
and dequantization operators to obtain the quantized model.
Args:
executor(paddle.static.Executor): The executor to load, run and save the
quantized model.
place(paddle.CPUPlace or paddle.CUDAPlace): This parameter represents
the executor run on which device.
model_dir(str): The path of fp32 model that will be quantized, and
the model and params that saved by ``paddle.static.io.save_inference_model``
are under the path.
quantize_model_path(str): The path to save quantized model using api
``paddle.static.io.save_inference_model``.
train_sample_generator(Python Generator): The sample generator provides
calibrate data for DataLoader, and it only returns a sample every time.
eval_sample_generator(Python Generator): The sample generator provides
evalution data for DataLoader, and it only returns a sample every time.
model_filename(str, optional): The name of model file. If parameters
are saved in separate files, set it as 'None'. Default: 'None'.
params_filename(str, optional): The name of params file.
When all parameters are saved in a single file, set it
as filename. If parameters are saved in separate files,
set it as 'None'. Default : 'None'.
save_model_filename(str): The name of model file to save the quantized inference program. Default: '__model__'.
save_params_filename(str): The name of file to save all related parameters.
If it is set None, parameters will be saved in separate files. Default: '__params__'.
scope(paddle.static.Scope, optional): The scope to run program, use it to load
and save variables. If scope is None, will use paddle.static.global_scope().
quantizable_op_type(list[str], optional): The list of op types
that will be quantized. Default: ["conv2d", "depthwise_conv2d",
"mul"].
is_full_quantize(bool): if True, apply quantization to all supported quantizable op type.
If False, only apply quantization to the input quantizable_op_type. Default is False.
weight_bits(int, optional): quantization bit number for weights.
activation_bits(int): quantization bit number for activation.
weight_quantize_type(str): quantization type for weights,
support 'abs_max' and 'channel_wise_abs_max'. Compared to 'abs_max',
the model accuracy is usually higher when using 'channel_wise_abs_max'.
optimize_model(bool, optional): If set optimize_model as True, it applies some
passes to optimize the model before quantization. So far, the place of
executor must be cpu it supports fusing batch_norm into convs.
is_use_cache_file(bool): This param is deprecated.
cache_dir(str): This param is deprecated.
runcount_limit(int): max. number of model quantization.
Returns:
None
"""
global g_quant_config
g_quant_config = QuantConfig(
executor, place, model_dir, quantize_model_path, algo, hist_percent,
bias_correct, batch_size, batch_num, train_sample_generator,
eval_sample_generator, train_dataloader, eval_dataloader,
eval_function, model_filename, params_filename, save_model_filename,
save_params_filename, scope, quantizable_op_type, is_full_quantize,
weight_bits, activation_bits, weight_quantize_type, optimize_model,
is_use_cache_file, cache_dir)
cs = ConfigurationSpace()
hyper_params = []
if 'hist' in algo:
hist_percent = UniformFloatHyperparameter(
"hist_percent",
hist_percent[0],
hist_percent[1],
default_value=hist_percent[0])
hyper_params.append(hist_percent)
if len(algo) > 1:
algo = CategoricalHyperparameter("algo", algo, default_value=algo[0])
hyper_params.append(algo)
else:
algo = algo[0]
if len(bias_correct) > 1:
bias_correct = CategoricalHyperparameter("bias_correct",
bias_correct,
default_value=bias_correct[0])
hyper_params.append(bias_correct)
else:
bias_correct = bias_correct[0]
if len(weight_quantize_type) > 1:
weight_quantize_type = CategoricalHyperparameter("weight_quantize_type", \
weight_quantize_type, default_value=weight_quantize_type[0])
hyper_params.append(weight_quantize_type)
else:
weight_quantize_type = weight_quantize_type[0]
if len(batch_size) > 1:
batch_size = UniformIntegerHyperparameter("batch_size",
batch_size[0],
batch_size[1],
default_value=batch_size[0])
hyper_params.append(batch_size)
else:
batch_size = batch_size[0]
if len(batch_num) > 1:
batch_num = UniformIntegerHyperparameter("batch_num",
batch_num[0],
batch_num[1],
default_value=batch_num[0])
hyper_params.append(batch_num)
else:
batch_num = batch_num[0]
if len(hyper_params) == 0:
quant_post( \
executor=g_quant_config.executor, \
scope=g_quant_config.scope, \
model_dir=g_quant_config.float_infer_model_path, \
quantize_model_path=g_quant_model_cache_path, \
sample_generator=g_quant_config.train_sample_generator, \
data_loader=g_quant_config.train_dataloader,
model_filename=g_quant_config.model_filename, \
params_filename=g_quant_config.params_filename, \
save_model_filename=g_quant_config.save_model_filename, \
save_params_filename=g_quant_config.save_params_filename, \
quantizable_op_type=g_quant_config.quantizable_op_type, \
activation_quantize_type="moving_average_abs_max", \
weight_quantize_type=weight_quantize_type, \
algo=algo, \
hist_percent=hist_percent, \
bias_correction=bias_correct, \
batch_size=batch_size, \
batch_nums=batch_num)
return
cs.add_hyperparameters(hyper_params)
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit":
runcount_limit, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "True",
"limit_resources": "False",
"memory_limit":
4096 # adapt this to reasonable value for your hardware
})
# To optimize, we pass the function to the SMAC-object
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=quantize)
# Example call of the function with default values
# It returns: Status, Cost, Runtime, Additional Infos
def_value = smac.get_tae_runner().run(cs.get_default_configuration(), 1)[1]
print("Value for default configuration: %.8f" % def_value)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
inc_value = smac.get_tae_runner().run(incumbent, 1)[1]
print("Optimized Value: %.8f" % inc_value)
print("quantize completed")
def run_experiment(out_path: str, on_travis: bool = False):
out_path = Path(out_path)
out_path.mkdir(exist_ok=True)
benchmark = Benchmark(container_source='library://phmueller/automl',
container_name='cartpole',
rng=1)
cs = benchmark.get_configuration_space(seed=1)
scenario_dict = {
"run_obj": "quality", # we optimize quality (alternative to runtime)
"wallclock-limit":
5 * 60 * 60, # max duration to run the optimization (in seconds)
"cs": cs, # configuration space
"deterministic": "true",
"runcount-limit": 200,
"limit_resources": True, # Uses pynisher to limit memory and runtime
"cutoff": 1800, # runtime limit for target algorithm
"memory_limit":
10000, # adapt this to reasonable value for your hardware
"output_dir": str(out_path),
}
if on_travis:
scenario_dict.update(get_travis_settings('smac'))
scenario = Scenario(scenario_dict)
# Number of Agents, which are trained to solve the cartpole experiment
max_budget = 5 if not on_travis else 2
def optimization_function_wrapper(cfg, seed, instance, budget):
""" Helper-function: simple wrapper to use the benchmark with smac """
b = Benchmark(container_source='library://phmueller/automl',
container_name='cartpole',
rng=seed)
# Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities
# result_dict = b.objective_function(cfg, budget=int(budget))
# New API ---- Use this
result_dict = b.objective_function(cfg,
fidelity={"budget": int(budget)})
return result_dict['function_value']
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=optimization_function_wrapper,
intensifier=SuccessiveHalving,
intensifier_kwargs={
'initial_budget': 1,
'max_budget': max_budget,
'eta': 3
})
start_time = time()
# Example call of the function with default values. It returns: Status, Cost, Runtime, Additional Infos
def_value = smac.get_tae_runner().run(
config=cs.get_default_configuration(), instance='1', budget=1,
seed=0)[1]
print(
f"Value for default configuration: {def_value:.4f}.\nEvaluation took {time() - start_time:.0f}s"
)
# Start optimization
start_time = time()
try:
smac.optimize()
finally:
incumbent = smac.solver.incumbent
end_time = time()
if not on_travis:
inc_value = smac.get_tae_runner().run(config=incumbent,
instance='1',
budget=max_budget,
seed=0)[1]
print(f"Value for optimized configuration: {inc_value:.4f}.\n"
f"Optimization took {end_time-start_time:.0f}s")
def smac_validation(
model_function,
term_library,
X_train,
U_train,
X_eval,
U_eval,
n_iter,
use_regularization,
):
def evaluation_function(params: dict, instance, budget, **kwargs):
terms = []
for idx in range(len(term_library)):
if params[str(idx)] > 0.5:
terms.append(term_library[int(idx)])
if use_regularization:
regularization = params["reg"]
else:
regularization = 1.0
errors = []
model = model_function(terms, regularization)
model.train_BFGS(X_train, U_train)
U_hat = model.predict(X_eval)
errors.append(rmse(U_eval, U_hat))
model.cleanup()
# Minimize
best_error = np.min(errors)
return best_error
terms = []
for i in range(len(term_library)):
terms.append(UniformIntegerHyperparameter(
str(i),
0,
1,
))
cs = ConfigurationSpace()
cs.add_hyperparameters(terms)
if use_regularization:
cs.add_hyperparameter(
UniformFloatHyperparameter("reg", 1e-5, 100, log=True))
scenario = Scenario({
"run_obj": "quality",
"cs": cs,
"runcount-limit": n_iter,
"limit_resources": False,
"deterministic": True,
})
smac = SMAC4HPO(scenario=scenario, tae_runner=evaluation_function)
incumbent = smac.optimize()
return smac, incumbent
# You can define individual crash costs for each objective
"cost_for_crash": [1, float(MAXINT)],
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default config's cost: {cost:2f}, training time: {time:2f} seconds".
format(**def_value))
# Optimize, using a SMAC-object
print(
"Optimizing! Depending on your machine, this might take a few minutes."
)
# Pass the multi objective algorithm and its hyperparameters
smac = SMAC4HPO(
scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=svm_from_cfg,
multi_objective_algorithm=ParEGO,
multi_objective_kwargs={
"rho": 0.05,
},
)
incumbent = smac.optimize()
# pareto front based on smac.runhistory.data
cost = np.vstack([v[0] for v in smac.runhistory.data.values()])
plot_pareto_from_runhistory(cost)
print('#' * 80)
print('#' * 80)
# SMAC scenario object
scenario = Scenario({
"run_obj":
"quality",
"runcount-limit":
25,
"cs":
cs,
"deterministic":
"true",
"output-dir":
"/home/feurerm/projects/smac3parallel/new/%s" % str(n_workers),
})
# To optimize, we pass the function to the SMAC-object
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(seed),
tae_runner=branin.branin
if n_workers == 1 else branin.branin_sleep,
n_jobs=n_workers,
intensifier=SimpleIntensifier)
incumbent = smac.optimize()
del smac
time.sleep(1)
sys.stdout.flush()
sys.stderr.flush()
# "initial_incumbent": "DEFAULT",
})
i = int(sys.argv[1])
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
if i == 1:
initial_configuration = [cs.get_default_configuration()]
else:
initial_configuration = None
tae_runner = partial(gadma_from_cfg, i)
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(i),
tae_runner=tae_runner,
run_id=i,
intensifier_kwargs=intensifier_kwargs,
initial_design=None,
initial_configurations=initial_configuration)
incumbent = smac.optimize()
def_costs = []
for i in INSTANCES:
cost = smac.get_tae_runner().run(cs.get_default_configuration(), i[0])[1]
def_costs.append(cost)
print("Value for default configuration: %.4f" % (np.mean(def_costs)))
inc_costs = []
for i in INSTANCES:
cost = smac.get_tae_runner().run(incumbent, i[0])[1]
tae_runner=tat,
pca_components=2)
hpo_result, info = run_smac_based_optimizer(hpo, tae)
write_output(
f"[{name}] time={info['time']} train_loss={info['last_train_loss']} "
f"test_loss={info['last_test_loss']}\n")
records = util.add_record(records, task_id, name, hpo_result)
########################################################################################################
# SMAC
########################################################################################################
name = "smac"
print(f"\n[{name}] ")
hpo = SMAC4HPO(scenario=scenario, rng=rng, tae_runner=tat)
hpo_result, info = run_smac_based_optimizer(hpo, tae)
write_output(
f"[{name}] time={info['time']} train_loss={info['last_train_loss']} "
f"test_loss={info['last_test_loss']}\n")
records = util.add_record(records, task_id, name, hpo_result)
########################################################################################################
# ROAR x2
########################################################################################################
name = "roar_x2"
print(f"\n[{name}] ")
hpo = ROAR(scenario=scenario, rng=rng, tae_runner=tat)
hpo_result, info = run_smac_based_optimizer(hpo, tae, speed=2)
values, configs = readData(cs, "hydrogen_new_3.csv")
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": len(values) + 1,
"cs": cs,
"deterministic": "true",
"limit-resources": "false"
})
for _ in range(100):
smac = SMAC4HPO(
scenario=scenario,
rng=np.random.RandomState(random.randint(0, 1000000)),
tae_runner=optFunc(values),
# acquisition_function=EI,
acquisition_function_kwargs={'par': 0.5},
# runhistory2epm=RunHistory2EPM4LogCost,
initial_design=None,
initial_configurations=list(values.keys()),
)
try:
smac.optimize()
except Exception:
pass
data = []
for config in results.keys():
res = dict(config._values)
res['flow'] = results[config]
data.append(res)
data.sort(key=lambda d: -d['flow'])
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=-4)
cs.add_hyperparameters([x0, x1])
# Scenario object
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 10, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = rosenbrock_2d(cs.get_default_configuration())
print("Default Value: %.2f" % def_value)
# Optimize, using a SMAC-object
for acquisition_func in (LCB, EI, PI):
print("Optimizing with %s! Depending on your machine, this might take a few minutes." % acquisition_func)
smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42),
tae_runner=rosenbrock_2d,
initial_design=LHDesign,
initial_design_kwargs={'n_configs_x_params': 4,
'max_config_fracs': 1.0},
runhistory2epm=RunHistory2EPM4InvScaledCost,
acquisition_function_optimizer_kwargs={'max_steps': 100},
acquisition_function=acquisition_func,
acquisition_function_kwargs={'par': 0.01}
)
smac.optimize()
os.system(cmd)
cmd = 'sh ' + '__run 1'
os.system(cmd)
cmd = 'echo "time ./a.out $cmd" > tmp_bat'
os.system(cmd)
begin = time.time()
for i in range(100):
os.system('source tmp_bat')
end = time.time()
return end-begin
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 10, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42),
tae_runner=run_gcc_time)
incumbent = smac.optimize()
inc_value = run_gcc_time(incumbent)
print ('the best time!')
print (inc_value)
print ('-----------------------------')
print ('-----------------------------')
with open('best_time.txt', 'w') as f:
f.write(str(inc_value))
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality
# (alternatively runtime)
"runcount-limit": 10,
"cs": cs, # configuration space
"deterministic": "false",
"shared_model": True,
"input_psmac_dirs": "smac-output-maxEnt",
"cutoff_time": 9000,
"wallclock_limit": 'inf'
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = maxEnt(cs.get_default_configuration())
print("Default Value: %.2f" % def_value)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=maxEnt)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
def test_ta_integration_to_smbo(self):
"""
In SMBO. 3 objects need to actively comunicate:
-> stats
-> epm
-> runhistory
This method makes sure that executed jobs are properly registered
in the above objects
It uses n_workers to test parallel and serial implementations!!
"""
for n_workers in range(1, 2):
# We create a controlled setting, in which we optimize x^2
# This will allow us to make sure every component act as expected
# FIRST: config space
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter("x", -10.0, 10.0))
smac = SMAC4HPO(
scenario=Scenario({
"n_workers": n_workers,
"cs": cs,
"runcount_limit": 5,
"run_obj": "quality",
"deterministic": True,
"limit_resources": True,
"initial_incumbent": "DEFAULT",
"output_dir": "data-test_smbo",
}),
tae_runner=ExecuteTAFuncArray,
tae_runner_kwargs={"ta": target},
)
# Register output dir for deletion
self.output_dirs.append(smac.output_dir)
smbo = smac.solver
# SECOND: Intensifier that tracks configs
all_configs = []
def mock_get_next_run(**kwargs):
config = cs.sample_configuration()
all_configs.append(config)
return (
RunInfoIntent.RUN,
RunInfo(
config=config,
instance=time.time() % 10,
instance_specific={},
seed=0,
cutoff=None,
capped=False,
budget=0.0,
),
)
intensifier = unittest.mock.Mock()
intensifier.num_run = 0
intensifier.process_results.return_value = (0.0, 0.0)
intensifier.get_next_run = mock_get_next_run
smac.solver.intensifier = intensifier
# THIRD: Run in this controlled setting
smbo.run()
# FOURTH: Checks
# Make sure all configs where launched
self.assertEqual(len(all_configs), 5)
# Run history
for k, v in smbo.runhistory.data.items():
# All configuration should be successful
self.assertEqual(v.status, StatusType.SUCCESS)
# The value should be the square version of the config
# The runhistory has config_ids = {config: int}
# The k here is {config_id: int}. We search for the actual config
# by inverse searching this runhistory.config dict
config = list(smbo.runhistory.config_ids.keys())[list(
smbo.runhistory.config_ids.values()).index(k.config_id)]
self.assertEqual(v.cost, config.get("x")**2)
# No config is lost in the config history
self.assertCountEqual(smbo.runhistory.config_ids.keys(),
all_configs)
# Stats!
# We do not exceed the number of target algorithm runs
self.assertEqual(smbo.stats.submitted_ta_runs, len(all_configs))
self.assertEqual(smbo.stats.finished_ta_runs, len(all_configs))
# No config is lost
self.assertEqual(smbo.stats.n_configs, len(all_configs))
# The EPM can access all points. This is something that
# also relies on the runhistory
X, Y, X_config = smbo.epm_chooser._collect_data_to_train_model()
self.assertEqual(X.shape[0], len(all_configs))
def main_cli(
self,
commandline_arguments: typing.Optional[typing.List[str]] = None
) -> None:
"""Main function of SMAC for CLI interface"""
self.logger.info("SMAC call: %s" % (" ".join(sys.argv)))
cmd_reader = CMDReader()
kwargs = {}
if commandline_arguments:
kwargs['commandline_arguments'] = commandline_arguments
main_args_, smac_args_, scen_args_ = cmd_reader.read_cmd(**kwargs)
root_logger = logging.getLogger()
root_logger.setLevel(main_args_.verbose_level)
logger_handler = logging.StreamHandler(stream=sys.stdout)
if root_logger.level >= logging.INFO:
formatter = logging.Formatter("%(levelname)s:\t%(message)s")
else:
formatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(name)s:\t%(message)s",
"%Y-%m-%d %H:%M:%S")
logger_handler.setFormatter(formatter)
root_logger.addHandler(logger_handler)
# remove default handler
if len(root_logger.handlers) > 1:
root_logger.removeHandler(root_logger.handlers[0])
# Create defaults
rh = None
initial_configs = None
stats = None
incumbent = None
# Create scenario-object
scenario = {}
scenario.update(vars(smac_args_))
scenario.update(vars(scen_args_))
scen = Scenario(scenario=scenario)
# Restore state
if main_args_.restore_state:
root_logger.debug("Restoring state from %s...",
main_args_.restore_state)
restore_state = main_args_.restore_state
rh, stats, traj_list_aclib, traj_list_old = self.restore_state(
scen, restore_state)
scen.output_dir_for_this_run = create_output_directory(
scen,
main_args_.seed,
root_logger,
)
scen.write()
incumbent = self.restore_state_after_output_dir(
scen, stats, traj_list_aclib, traj_list_old)
if main_args_.warmstart_runhistory:
rh = RunHistory()
scen, rh = merge_foreign_data_from_file(
scenario=scen,
runhistory=rh,
in_scenario_fn_list=main_args_.warmstart_scenario,
in_runhistory_fn_list=main_args_.warmstart_runhistory,
cs=scen.cs, # type: ignore[attr-defined] # noqa F821
)
if main_args_.warmstart_incumbent:
initial_configs = [scen.cs.get_default_configuration()
] # type: ignore[attr-defined] # noqa F821
for traj_fn in main_args_.warmstart_incumbent:
trajectory = TrajLogger.read_traj_aclib_format(
fn=traj_fn,
cs=scen.cs, # type: ignore[attr-defined] # noqa F821
)
initial_configs.append(trajectory[-1]["incumbent"])
if main_args_.mode == "SMAC4AC":
optimizer = SMAC4AC(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "SMAC4HPO":
optimizer = SMAC4HPO(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "SMAC4BB":
optimizer = SMAC4BB(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "ROAR":
optimizer = ROAR(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
run_id=main_args_.seed)
elif main_args_.mode == "Hydra":
optimizer = Hydra(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed,
random_configuration_chooser=main_args_.
random_configuration_chooser,
n_iterations=main_args_.hydra_iterations,
val_set=main_args_.hydra_validation,
incs_per_round=main_args_.hydra_incumbents_per_round,
n_optimizers=main_args_.hydra_n_optimizers)
elif main_args_.mode == "PSMAC":
optimizer = PSMAC(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
run_id=main_args_.seed,
shared_model=smac_args_.shared_model,
validate=main_args_.psmac_validate,
n_optimizers=main_args_.hydra_n_optimizers,
n_incs=main_args_.hydra_incumbents_per_round,
)
try:
optimizer.optimize()
except (TAEAbortException, FirstRunCrashedException) as err:
self.logger.error(err)
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit":
500, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = LR_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=LR_from_cfg)
a_time = time.process_time()
incumbent = smac.optimize()
b_time = time.process_time()
print("+++++++++++++++++++++++")
print("Optimization finished. CPU time consumed: %s" % (a_time - b_time))
print("+++++++++++++++++++++++")
inc_value = LR_from_cfg(incumbent)
print("Optimized Value: %.6f" % (inc_value))
print("before validate time:%s" %
cs.add_hyperparameters([learning_rate, step_size_scalar, l2_regularizer, N])
# Scenario object
scenario = Scenario({"run_obj": "quality", # we optimize quality
# (alternatively runtime)
"runcount-limit": 2,
"cs": cs, # configuration space
"deterministic": "false",
"shared_model": True,
"input_psmac_dirs": "smac-output-learch",
"cutoff_time": 9000,
"wallclock_limit": 'inf'
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = learch_variant(cs.get_default_configuration())
print("Default Value: %.2f" % def_value)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=learch_variant)
# Start optimization
try:
incumbent = smac.optimize()
finally:
incumbent = smac.solver.incumbent
